.github/workflows/benchmark.yml

name: Benchmark
on:
  pull_request:
    paths:
      - ".github/workflows/benchmark.yml"
      - "src/**"
      - "tests/**"
  workflow_dispatch:
    inputs:
      base_commit:
        description: The base commit to compare against

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  benchmark:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        timeout-minutes: 3
        with:
          persist-credentials: false
          path: "cryptography-pr"
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        timeout-minutes: 3
        with:
          persist-credentials: false
          repository: "pyca/cryptography"
          path: "cryptography-base"
          ref: "${{ github.event.inputs.base_commit || github.base_ref }}"
      - name: Clone test vectors
        timeout-minutes: 2
        uses: ./cryptography-base/.github/actions/fetch-vectors

      - name: Setup python
        id: setup-python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.11"

      - name: Create virtualenv (base)
        run: |
          python -m venv .venv-base
          .venv-base/bin/pip install -v -c ./cryptography-base/ci-constraints-requirements.txt "./cryptography-base[test]" ./cryptography-base/vectors/
      - name: Create virtualenv (PR)
        run: |
          python -m venv .venv-pr
          .venv-pr/bin/pip install -v -c ./cryptography-pr/ci-constraints-requirements.txt "./cryptography-pr[test]" ./cryptography-pr/vectors/

      - name: Run benchmarks (base)
        run: .venv-base/bin/pytest --benchmark-enable --benchmark-only ./cryptography-pr/tests/bench/ --benchmark-json=bench-base.json --x509-limbo-root=x509-limbo/
      - name: Run benchmarks (PR)
        run: .venv-pr/bin/pytest --benchmark-enable --benchmark-only ./cryptography-pr/tests/bench/ --benchmark-json=bench-pr.json --x509-limbo-root=x509-limbo/

      - name: Compare results
        run: python ./cryptography-pr/.github/bin/compare_benchmarks.py bench-base.json bench-pr.json | tee -a $GITHUB_STEP_SUMMARY