workflows/contrib-llm-test.yml

CI / automation

# REFERENCE-ONLY: GitHub Actions secrets referenced below are illustrative.
# Do not inject real credentials. See your CI/CD provider for proper secret management.
# This workflow will install Python dependencies and run tests
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Contrib tests with LLMs

on:
  schedule:
    - cron: "0 0 * * 0" # weekly on Sunday at midnight UTC
  workflow_dispatch: # allows manual triggering of the workflow
  # pull_request_target:
  #   branches: ["main"]
  #   paths:
  #     - "autogen/**"
  #     - "test/agentchat/contrib/**"
  #     - ".github/workflows/contrib-llm-test.yml"
  #     - "pyproject.toml"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
  cancel-in-progress: true
permissions:
  {}

jobs:
  TeachableAgent:
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.11"]
    runs-on: ${{ matrix.os }}
    environment: openai1
    steps:
      - name: Get User Permission
        id: checkAccess
        uses: actions-cool/check-user-permission@v2
        with:
          require: write
          username: ${{ github.triggering_actor }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Check User Permission
        if: steps.checkAccess.outputs.require-result == 'false'
        run: |
          echo "${{ github.triggering_actor }} does not have permissions on this repo."
          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
          echo "Job originally triggered by ${{ github.actor }}"
          exit 1
      # checkout to pr branch
      - name: Checkout
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: astral-sh/setup-uv@v6
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies
        run: |
          docker --version
          uv pip install --system -e .[test,openai,teachable]
      - name: Run tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
          OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
        run: |
          bash scripts/test.sh test/agentchat/contrib/capabilities/test_teachable_agent.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: teachable, ${{ matrix.python-version }}, ${{ matrix.os }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  AgentBuilder:
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.11"]
    runs-on: ${{ matrix.os }}
    environment: openai1
    steps:
      - name: Get User Permission
        id: checkAccess
        uses: actions-cool/check-user-permission@v2
        with:
          require: write
          username: ${{ github.triggering_actor }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Check User Permission
        if: steps.checkAccess.outputs.require-result == 'false'
        run: |
          echo "${{ github.triggering_actor }} does not have permissions on this repo."
          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
          echo "Job originally triggered by ${{ github.actor }}"
          exit 1
      # checkout to pr branch
      - name: Checkout
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: astral-sh/setup-uv@v6
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies
        run: |
          docker --version
          uv pip install --system -e ".[test,openai,autobuild,captainagent,gemini]"
      - name: Create OAI_CONFIG_LIST from Secret
        run: |
          cat > OAI_CONFIG_LIST << 'EOF'
          ${{ secrets.OAI_CONFIG_LIST }}
          EOF
        shell: bash
      - name: Run tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
          OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
        run: |
          bash scripts/test.sh test/agentchat/contrib/test_agent_builder.py test/agentchat/contrib/test_captainagent.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: autobuild, captainagent, ${{ matrix.python-version }}, ${{ matrix.os }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  ImageGen:
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.13"]
    runs-on: ${{ matrix.os }}
    environment: openai1
    steps:
      - name: Get User Permission
        id: checkAccess
        uses: actions-cool/check-user-permission@v2
        with:
          require: write
          username: ${{ github.triggering_actor }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Check User Permission
        if: steps.checkAccess.outputs.require-result == 'false'
        run: |
          echo "${{ github.triggering_actor }} does not have permissions on this repo."
          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
          echo "Job originally triggered by ${{ github.actor }}"
          exit 1
      # checkout to pr branch
      - name: Checkout
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: astral-sh/setup-uv@v6
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies
        run: |
          docker --version
          uv pip install --system -e .[test,openai,lmm]
      - name: Run tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          bash scripts/test.sh test/agentchat/contrib/capabilities/test_image_generation_capability.py
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: lmm, ${{ matrix.python-version }}, ${{ matrix.os }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  # https://github.com/marketplace/actions/alls-green#why
  check-contrib-test-llm: # This job does nothing and is only used for the branch protection
    # from: https://github.com/re-actors/alls-green
    # Important: For this to work properly, it is a must to have the job always
    # run, otherwise GitHub will make it skipped when any of the dependencies
    # fail. In some contexts, skipped is interpreted as success which may lead
    # to undersired, unobvious and even dangerous (as in security breach
    # "dangerous") side-effects.
    if: always()

    needs:
      - TeachableAgent
      - AgentBuilder
      - ImageGen

    runs-on: ubuntu-latest

    steps:
      - name: Decide whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@release/v1 # nosemgrep
        with:
          # allowed-failures: docs, linters
          # allowed-skips: non-voting-flaky-job
          jobs: ${{ toJSON(needs) }}