skill_eval_harness: per-model API endpoint support #93
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Matrix | |
| on: | |
| pull_request: | |
| branches: [ master ] | |
| workflow_dispatch: | |
| inputs: | |
| upload_to_repo: | |
| description: "Also commit benchmark JSON outputs to benchmark-results/ in this repository" | |
| required: false | |
| default: false | |
| type: boolean | |
| permissions: | |
| contents: read | |
| jobs: | |
| benchmark: | |
| name: Benchmark (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| python-version: ["3.12"] | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install eth-keyfile==0.9.1 | |
| - name: Run benchmark | |
| run: | | |
| python -c "import os; os.makedirs('benchmark-results', exist_ok=True)" | |
| python benchmark.py --duration 30 --comment "GitHub actions benchmark run on ${{ matrix.os }} (run ${{ github.run_id }})" --output "benchmark-results/benchmark_${{ matrix.os }}_${{ github.run_id }}.json" | |
| - name: Upload benchmark artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-${{ matrix.os }} | |
| path: benchmark-results/benchmark_${{ matrix.os }}_${{ github.run_id }}.json | |
| if-no-files-found: error | |
| upload-to-repo: | |
| name: Upload benchmark outputs to repo | |
| runs-on: ubuntu-latest | |
| needs: benchmark | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.upload_to_repo }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Download benchmark artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: benchmark-* | |
| path: /tmp/benchmark-artifacts | |
| merge-multiple: true | |
| - name: Copy outputs into benchmark-results/ | |
| run: | | |
| mkdir -p benchmark-results | |
| cp /tmp/benchmark-artifacts/*.json benchmark-results/ | |
| - name: Commit and push benchmark outputs | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add benchmark-results/*.json | |
| if git --no-pager diff --cached --quiet; then | |
| echo "No new benchmark output files to commit." | |
| exit 0 | |
| fi | |
| git commit -m "Add benchmark outputs from GitHub Actions run ${{ github.run_id }}" | |
| git push |