Integration Tests (live) #43
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests (live) | |
| # Only run on demand or on a nightly schedule. PR + main-branch pushes run the | |
| # fast unit suite from `ci.yml` — they do NOT call provider APIs (cost + flake). | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| providers: | |
| description: 'Comma-separated provider keys (e.g. "openai,anthropic"); blank = all' | |
| required: false | |
| default: '' | |
| schedule: | |
| - cron: '0 2 * * *' # 02:00 UTC nightly | |
| env: | |
| PYTHON_VERSION: "3.12" | |
| jobs: | |
| matrix: | |
| name: ${{ matrix.provider }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| # Don't fail the whole run if one provider is having an outage. | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Each entry runs one provider's live integration tests. Providers | |
| # with a dedicated ``test_<provider>_integration.py`` file get the | |
| # full per-provider suite. Providers without a dedicated file | |
| # (Tier-A wrappers / cloud platforms added in 0.4) run a filtered | |
| # slice of ``test_provider_matrix.py`` via ``args``. | |
| include: | |
| - { provider: openai, env: OPENAI_API_KEY, file: tests/integration/test_openai_integration.py } | |
| - { provider: anthropic, env: ANTHROPIC_API_KEY, file: tests/integration/test_anthropic_integration.py } | |
| - { provider: gemini, env: GEMINI_API_KEY, file: tests/integration/test_gemini_integration.py } | |
| - { provider: mistral, env: MISTRAL_API_KEY, file: tests/integration/test_mistral_integration.py } | |
| - { provider: cohere, env: COHERE_API_KEY, file: tests/integration/test_cohere_integration.py } | |
| - { provider: groq, env: GROQ_API_KEY, file: tests/integration/test_groq_integration.py } | |
| - { provider: together, env: TOGETHER_API_KEY, file: tests/integration/test_together_integration.py } | |
| - { provider: fireworks, env: FIREWORKS_API_KEY, file: tests/integration/test_fireworks_integration.py } | |
| - { provider: deepseek, env: DEEPSEEK_API_KEY, file: tests/integration/test_deepseek_integration.py } | |
| - { provider: perplexity, env: PERPLEXITY_API_KEY, file: tests/integration/test_perplexity_integration.py } | |
| # 0.4 production push — no dedicated suite; run the cross-provider | |
| # matrix file filtered to this provider's name. | |
| - { provider: xai, env: XAI_API_KEY, file: tests/integration/test_provider_matrix.py, args: "-k xai" } | |
| - { provider: openrouter, env: OPENROUTER_API_KEY, file: tests/integration/test_provider_matrix.py, args: "-k openrouter" } | |
| - { provider: cerebras, env: CEREBRAS_API_KEY, file: tests/integration/test_provider_matrix.py, args: "-k cerebras" } | |
| - { provider: sambanova, env: SAMBANOVA_API_KEY, file: tests/integration/test_provider_matrix.py, args: "-k sambanova" } | |
| - { provider: deepinfra, env: DEEPINFRA_API_KEY, file: tests/integration/test_provider_matrix.py, args: "-k deepinfra" } | |
| - { provider: nvidia_nim, env: NVIDIA_NIM_API_KEY, file: tests/integration/test_provider_matrix.py, args: "-k nvidia_nim" } | |
| - { provider: bedrock, env: AWS_ACCESS_KEY_ID, file: tests/integration/test_provider_matrix.py, args: "-k bedrock" } | |
| - { provider: watsonx, env: WATSONX_API_KEY, file: tests/integration/test_provider_matrix.py, args: "-k watsonx" } | |
| # Cross-provider agentic-parity matrix — one row per (provider, chat | |
| # model) for streaming, tools, structured output, reasoning. Gates on | |
| # OPENAI_API_KEY so the job is skipped on PRs from forks; per-row | |
| # tests skip individually when their provider's key is missing. | |
| - { provider: agentic-parity, env: OPENAI_API_KEY, file: tests/integration/test_agentic_parity.py } | |
| steps: | |
| - name: Skip when secret missing | |
| id: gate | |
| run: | | |
| # Per-provider key check. Skip the rest of the job if the secret isn't configured — | |
| # this lets contributors run the workflow without paying for every provider. | |
| if [[ -z "${{ secrets[matrix.env] }}" ]]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "::notice::No ${{ matrix.env }} secret configured; skipping ${{ matrix.provider }}." | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Skip when filter excludes provider | |
| if: steps.gate.outputs.skip == 'false' && inputs.providers != '' | |
| id: filter | |
| run: | | |
| if [[ ",${{ inputs.providers }}," != *",${{ matrix.provider }},"* ]]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "::notice::${{ matrix.provider }} excluded by 'providers' input." | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/checkout@v4 | |
| if: steps.gate.outputs.skip != 'true' && steps.filter.outputs.skip != 'true' | |
| - uses: actions/setup-python@v5 | |
| if: steps.gate.outputs.skip != 'true' && steps.filter.outputs.skip != 'true' | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: Install | |
| if: steps.gate.outputs.skip != 'true' && steps.filter.outputs.skip != 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Run live integration tests | |
| if: steps.gate.outputs.skip != 'true' && steps.filter.outputs.skip != 'true' | |
| env: | |
| # Each provider job gets the full secret set so cross-provider | |
| # tests (e.g. test_provider_matrix.py) can resolve any key. Per- | |
| # provider integration files only read the one secret that's | |
| # gated above. | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} | |
| COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} | |
| FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} | |
| XAI_API_KEY: ${{ secrets.XAI_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} | |
| SAMBANOVA_API_KEY: ${{ secrets.SAMBANOVA_API_KEY }} | |
| DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} | |
| NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }} | |
| WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }} | |
| WATSONX_URL: ${{ secrets.WATSONX_URL }} | |
| WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} | |
| run: | | |
| pytest "${{ matrix.file }}" ${{ matrix.args || '' }} \ | |
| -v --tb=short --timeout=60 \ | |
| --junitxml=junit-${{ matrix.provider }}.xml | |
| - name: Upload junit report | |
| if: always() && steps.gate.outputs.skip != 'true' && steps.filter.outputs.skip != 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-${{ matrix.provider }} | |
| path: junit-${{ matrix.provider }}.xml | |
| if-no-files-found: ignore | |
| parallel-smoke: | |
| name: All providers in parallel | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| continue-on-error: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Run cross-provider parallel smoke | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} | |
| COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} | |
| FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} | |
| XAI_API_KEY: ${{ secrets.XAI_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} | |
| SAMBANOVA_API_KEY: ${{ secrets.SAMBANOVA_API_KEY }} | |
| DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} | |
| NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }} | |
| WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }} | |
| WATSONX_URL: ${{ secrets.WATSONX_URL }} | |
| WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} | |
| run: | | |
| pytest tests/integration/test_provider_matrix.py \ | |
| -v --tb=short --timeout=120 \ | |
| --junitxml=junit-parallel.xml | |
| - if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: parallel-smoke | |
| path: junit-parallel.xml | |
| if-no-files-found: ignore |