workflows/test-with-optional-deps.yml

CI / automation

# REFERENCE-ONLY: GitHub Actions secrets referenced below are illustrative.
# Do not inject real credentials. See your CI/CD provider for proper secret management.
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Tests with optional dependencies

on:
  schedule:
    - cron: "0 0 * * *" # daily at midnight UTC
  workflow_dispatch: # allows manual triggering of the workflow
  # pull_request:
  #   branches: ["main"]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
  cancel-in-progress: true
permissions: {}

jobs:
  paths-filter:
    runs-on: ubuntu-latest
    outputs:
      hasChanges: ${{ steps.filter.outputs.autogen == 'true' || steps.filter.outputs.test == 'true' || steps.filter.outputs.workflows == 'true' || steps.filter.outputs.setup == 'true' }}
    steps:
      - uses: actions/checkout@v6
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            autogen:
              - "autogen/**"
            test:
              - "test/**"
            workflows:
              - ".github/workflows/**"
            setup:
              - "pyproject.toml"
      - name: autogen has changes
        run: echo "autogen has changes"
        if: steps.filter.outputs.autogen == 'true'
      - name: test has changes
        run: echo "test has changes"
        if: steps.filter.outputs.test == 'true'
      - name: workflows has changes
        run: echo "workflows has changes"
        if: steps.filter.outputs.workflows == 'true'
      - name: setup has changes
        run: echo "setup has changes"
        if: steps.filter.outputs.setup == 'true'

  test:
    needs: paths-filter
    if: needs.paths-filter.outputs.hasChanges == 'true'
    runs-on: ${{ matrix.os }}
    env:
      AUTOGEN_USE_DOCKER: 'True'  # Evokore: always use Docker for safety
    strategy:
      fail-fast: false
      matrix:
        optional-dependencies:
          - "browser-use"
          - "commsagent-discord"
          - "commsagent-slack"
          - "commsagent-telegram"
          - "jupyter-executor"
          - "rag"
          - "retrievechat"
          - "retrievechat-pgvector"
          - "retrievechat-mongodb"
          - "retrievechat-qdrant"
          - "graph-rag-falkor-db"
          # - "neo4j"  # Separate test exists at contrib-graph-rag-tests.yml
          - "twilio"
          - "interop"
          - "crawl4ai"
          - "docs"
          - "interop-crewai"
          - "interop-langchain"
          - "interop-pydantic-ai"
          - "mcp"
          - "websockets"
          - "google-api"
          - "wikipedia-api"
        os: [ubuntu-latest, macos-latest, windows-latest]
        # Run tests only for py versions 3.10, 3.13
        python-version: ["3.10", "3.13"]
        # add more if needed
        exclude:
          # Issues with installing retrievechat-qdrant in python 3.13
          - python-version: "3.13"
            optional-dependencies: "retrievechat-qdrant"
          # Browser use doesn't support below python 3.11
          - python-version: "3.10"
            optional-dependencies: "browser-use"
          # Skip if os is mac or windows, python version is 3.13 and optional-dependencies is starting with retrievechat
          # Because torch and sentence_transformers creates issues while trying to install
          - os: macos-latest
            python-version: "3.13"
            optional-dependencies: "retrievechat"
          - os: macos-latest
            python-version: "3.13"
            optional-dependencies: "retrievechat-pgvector"
          - os: macos-latest
            python-version: "3.13"
            optional-dependencies: "retrievechat-mongodb"
          - os: windows-latest
            python-version: "3.13"
            optional-dependencies: "retrievechat"
          - os: windows-latest
            python-version: "3.13"
            optional-dependencies: "retrievechat-pgvector"
          - os: windows-latest
            python-version: "3.13"
            optional-dependencies: "retrievechat-mongodb"
          # Skip RAG tests on macOS and Windows with Python 3.13 due to issues with dependencies like PyTorch and sentence_transformers
          - os: macos-latest
            python-version: "3.13"
            optional-dependencies: "rag"
          - os: windows-latest
            python-version: "3.13"
            optional-dependencies: "rag"
    steps:
      - uses: actions/checkout@v6
      - uses: astral-sh/setup-uv@v6
        with:
          version: "latest"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install packages and dependencies
        run: |
          uv pip install --system -e .[test,openai,${{ matrix.optional-dependencies }}]
      - name: Set AUTOGEN_USE_DOCKER based on OS
        shell: bash
        run: |
          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
            echo "AUTOGEN_USE_DOCKER=True" >> $GITHUB_ENV
          fi
      - name: Install playwright
        if: matrix.optional-dependencies == 'browser-use' || matrix.optional-dependencies == 'crawl4ai'
        run: |
          playwright install
      - name: Test import all submodules
        shell: bash
        run: |
          bash scripts/test.sh test/test_import.py
      - name: Test with pytest using optional dependency marks
        shell: bash
        run: |
          OPTIONAL_DEPENDENCIES="${{ matrix.optional-dependencies }}"
          MARKER="${OPTIONAL_DEPENDENCIES//-/_}"
          # openai mark actually means that we need OpenAI API key, not just the package installed
          echo pytest -m "(not (openai or openai_realtime or gemini or gemini_realtime or anthropic or deepseek or ollama or bedrock or cerebras) and ($MARKER))"
          pytest -m "(not (openai or openai_realtime or gemini or gemini_realtime or anthropic or deepseek or ollama or bedrock or cerebras) and ($MARKER))"
      - name: Show coverage report
        run: bash scripts/show-coverage-report.sh
      - name: Upload coverage to Codecov
        if: ${{ !contains(github.ref, 'gh-readonly-queue/') }}
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: optional-deps, ${{ matrix.optional-dependencies }}, ${{ matrix.os }}, ${{ matrix.python-version }}
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  test-with_optional-dependencies-check:
    if: always()
    runs-on: ubuntu-latest
    needs: [test]
    steps:
      - name: Get Date
        shell: bash
        run: |
          echo "date=$(date +'%m/%d/%Y %H:%M:%S')" >> "$GITHUB_ENV"

      - name: Run Type is ${{ github.event_name }}
        if: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'}}
        shell: bash
        run: |
          echo "run_type=${{ github.event_name }}" >> "$GITHUB_ENV"

      - name: Fail workflow if build failed
        id: check_build_failed
        if: contains(join(needs.*.result, ','), 'failure')
        uses: actions/github-script@v8
        with:
          script: core.setFailed('Build Failed!')

      - name: Fail workflow if build cancelled
        id: check_build_cancelled
        if: contains(join(needs.*.result, ','), 'cancelled')
        uses: actions/github-script@v8
        with:
          script: core.setFailed('Build Cancelled!')