workflows/core-llm-test.yml

CI / automation

# REFERENCE-ONLY: GitHub Actions secrets referenced below are illustrative.
# Do not inject real credentials. See your CI/CD provider for proper secret management.
# This workflow will install Python dependencies and run tests with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Core tests with LLMs

on:
  schedule:
    - cron: "0 0 * * 0" # weekly on Sunday at midnight UTC
  workflow_dispatch: # allows manual triggering of the workflow
  # pull_request_target:
  #   branches: ["main"]
  #   paths:
  #     - "autogen/**"
  #     - "test/**"
  #     - "notebook/agentchat_auto_feedback_from_code_execution.ipynb"
  #     - "notebook/agentchat_function_call.ipynb"
  #     - "notebook/agentchat_groupchat_finite_state_machine.ipynb"
  #     - ".github/workflows/core-llm-test.yml"
  #     - "pyproject.toml"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
  cancel-in-progress: true
permissions:
  {}

jobs:
  test:
    strategy:
      matrix:
        llm: ["openai", "openai-realtime", "gemini", "gemini-realtime", "anthropic", "deepseek", "ollama", "bedrock", "cerebras"]
        python-version: ["3.10"]
        os: [ubuntu-latest]
      fail-fast: false
    runs-on: ${{ matrix.os }}
    environment: openai1
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
        options: --entrypoint redis-server
    steps:
      - name: Get User Permission
        id: checkAccess
        uses: actions-cool/check-user-permission@v2
        with:
          require: write
          username: ${{ github.triggering_actor }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Check User Permission
        if: steps.checkAccess.outputs.require-result == 'false'
        run: |
          echo "${{ github.triggering_actor }} does not have permissions on this repo."
          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
          echo "Job originally triggered by ${{ github.actor }}"
          exit 1
      # checkout to pr branch
      - name: Checkout
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.pull_request.head.sha || github.ref }}
      - uses: astral-sh/setup-uv@v6
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages for ${{ matrix.llm }}
        run: |
          docker --version
          uv pip install --system -e ".[test,redis,${{ matrix.llm }}]"
      - name: Install openai if testing gemini-realtime
        if: ${{ matrix.llm == 'gemini-realtime' }}
        run: |
          uv pip install --system -e ".[openai]"
      - name: Create OAI_CONFIG_LIST from Secret
        run: |
          cat > OAI_CONFIG_LIST << 'EOF'
          ${{ secrets.OAI_CONFIG_LIST }}
          EOF
        shell: bash
      - name: LLM tests using ${{ matrix.llm }}
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
          OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
        run: |
          llm_fixed="${{ matrix.llm }}" # Use the matrix variable
          llm_fixed="${llm_fixed//-/_}" # Replace hyphen with underscore
          bash scripts/test-core-llm.sh -m "$llm_fixed"
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: core-llm, ${{ matrix.llm }}, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}