add CI to detect performance regressions

.github/workflows/perf-test.yml

@@ -0,0 +1,146 @@
+name: Performance Regression
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - cpp-linter/src/**
+      - cpp-linter/Cargo.toml
+      - Cargo.toml
+      - Cargo.lock
+      - .github/workflows/perf-test.yml
+      - .github/workflows/bench.py
+    tags-ignore: ['*']
+  pull_request:
+    branches: [main]
+    paths:
+      - cpp-linter/src/**
+      - cpp-linter/Cargo.toml
+      - Cargo.toml
+      - Cargo.lock
+      - .github/workflows/perf*
+jobs:
+  build:
+    name: Build ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - commit: ${{ github.sha }}
+            name: current
+          - commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
+            name: previous
+    outputs:
+      cached-previous: ${{ steps.is-cached-previous.outputs.is-cached == 'true' && steps.validate.outputs.cache-valid != 'false' }}
+      cached-current: ${{ steps.is-cached-current.outputs.is-cached == 'true' && steps.validate.outputs.cache-valid != 'false' }}
+    env:
+      BIN: target/release/cpp-linter
+    steps:
+      - name: Checkout ${{ matrix.name }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ matrix.commit }}
+      - name: Cache base ref build
+        uses: actions/cache@v4
+        id: cache
+        with:
+          key: bin-cache-${{ hashFiles('cpp-linter/src/**', 'Cargo.toml', 'Cargo.lock', 'cpp-linter/Cargo.toml') }}
+          path: ${{ env.BIN }}
+      - name: Is previous cached?
+        if: matrix.name == 'previous'
+        id: is-cached-previous
+        run: echo "is-cached=${{ steps.cache.outputs.cache-hit }}" >> "$GITHUB_OUTPUT"
+      - name: Is current cached?
+        if: matrix.name == 'current'
+        id: is-cached-current
+        run: echo "is-cached=${{ steps.cache.outputs.cache-hit }}" >> "$GITHUB_OUTPUT"
+      - name: Validate cached binary
+        if: steps.cache.outputs.cache-hit == 'true'
+        id: validate
+        run: |
+          chmod +x ${{ env.BIN }}
+          if ! ${{ env.BIN }} version; then
+            echo "Cached binary is invalid, rebuilding..."
+            echo "cache-valid=false" >> "$GITHUB_OUTPUT"
+          fi
+      - run: rustup update --no-self-update
+        if: steps.cache.outputs.cache-hit != 'true' || steps.validate.outputs.cache-valid == 'false'
+      - run: cargo build --bin cpp-linter --release
+        if: steps.cache.outputs.cache-hit != 'true' || steps.validate.outputs.cache-valid == 'false'
+      - name: Upload build artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.name }}
+          path: ${{ env.BIN }}
+
+  benchmark:
+    name: Measure Performance Difference
+    needs: [build]
+    if: ${{ !needs.build.outputs.cached-current || !needs.build.outputs.cached-previous }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Checkout libgit2
+        uses: actions/checkout@v4
+        with:
+          repository: libgit2/libgit2
+          ref: v1.8.1
+          path: libgit2
+      - name: Download built binaries
+        uses: actions/download-artifact@v4
+      - name: Make binaries executable
+        run: chmod +x ./*/cpp-linter
+      - name: Generate compilation database
+        working-directory: libgit2
+        run: |
+          mkdir build && cd build
+          cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
+      - name: Install cargo-binstall
+        uses: cargo-bins/cargo-binstall@main
+      - name: Install hyperfine
+        run: cargo binstall -y hyperfine
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: pip install 'cpp-linter < 2.0'
+      - name: Warmup and list files
+        env:
+          CPP_LINTER_COLOR: true
+        working-directory: libgit2
+        # Use previous build for stability. This will
+        # - create the .cpp-linter_cache folder
+        # - list the files concerning the benchmark test
+        # NOTE: This does not actually invoke clang tools.
+        run: ../previous/cpp-linter -l 0 -p build -i='|!src/libgit2' -s="" -c="-*" -e c
+      - name: Run hyperfine tool
+        # using the generated compilation database,
+        # we will use cpp-linter (both builds) to scan libgit2 src/libgit2/**.c files.
+        working-directory: libgit2
+        run: >-
+          hyperfine
+          --runs 2
+          --style color
+          --export-markdown '${{ runner.temp }}/benchmark.md'
+          --export-json '${{ runner.temp }}/benchmark.json'
+          --command-name=previous-build
+          "../previous/cpp-linter -l 0 -p build -i='|!src/libgit2' -e c"
+          --command-name=current-build
+          "../current/cpp-linter -l 0 -p build -i='|!src/libgit2' -e c"
+          --command-name=pure-python
+          "cpp-linter -l false -j 0 -p build -i='|!src/libgit2' -e c"
+      - name: Append report to job summary
+        run: cat ${{ runner.temp }}/benchmark.md >> "$GITHUB_STEP_SUMMARY"
+      - name: Upload JSON results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-json
+          path: ${{ runner.temp }}/benchmark.json
+      - name: Annotate summary
+        run: python .github/workflows/perf_annotate.py "${{ runner.temp }}/benchmark.json"
+
+  report-no-src-changes:
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: needs.build.outputs.cached-current && needs.build.outputs.cached-previous
+    steps:
+      - run: echo "::notice title=No benchmark performed::No changes to cpp-linter source code detected."

.github/workflows/perf_annotate.py

@@ -0,0 +1,68 @@
+import argparse
+import json
+from os import environ
+from pathlib import Path
+from typing import List, Any, Dict, cast
+
+
+class Args(argparse.Namespace):
+    json_file: Path
+
+
+def main():
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument("json_file", type=Path)
+    arg_parser.parse_args(namespace=Args)
+
+    bench_json = Args.json_file.read_text(encoding="utf-8")
+    bench: List[Dict[str, Any]] = json.loads(bench_json)["results"]
+
+    assert len(bench) == 3
+    old_mean, new_mean = (None, None)
+    for result in bench:
+        mean = cast(float, result["mean"])
+        if result["command"] == "previous-build":
+            old_mean = mean
+        elif result["command"] == "current-build":
+            new_mean = mean
+
+    assert old_mean is not None, "benchmark report has no result for previous-build"
+    assert new_mean is not None, "benchmark report has no result for current-build"
+
+    diff = round(new_mean - old_mean, 2)
+    scalar = int((new_mean - old_mean) / old_mean * 100)
+
+    output = []
+    if diff > 2:
+        output.extend(
+            [
+                "> [!CAUTION]",
+                "> Detected a performance regression in new changes:",
+            ]
+        )
+    elif diff < -2:
+        output.extend(
+            [
+                "> [!TIP]",
+                "> Detected a performance improvement in new changes:",
+            ]
+        )
+    else:
+        output.extend(
+            [
+                "> [!NOTE]",
+                "> Determined a negligible difference in performance with new changes:",
+            ]
+        )
+    output[-1] += f" {diff}s ({scalar} %)"
+    annotation = "\n".join(output)
+
+    if "GITHUB_STEP_SUMMARY" in environ:
+        with open(environ["GITHUB_STEP_SUMMARY"], "a") as summary:
+            summary.write(f"\n{annotation}\n")
+    else:
+        print(annotation)
+
+
+if __name__ == "__main__":
+    main()