ci-extensions.yml

name: ci-extensions

on:
  push:
    branches: [main]
  pull_request:

jobs:
  extensions-tests:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macOS-latest]
        python-version: ["3.10", "3.11"]

    steps:
      - name: Clean up instance space
        if: matrix.os != 'windows-latest'
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          df -h

      - uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Poetry (Unix)
        if: matrix.os != 'windows-latest'
        run: |
          curl -sSL https://install.python-poetry.org | python3 -
          echo 'export PATH="$HOME/.local/bin:$PATH"' >> $GITHUB_ENV

      - name: Install Poetry (Windows)
        if: matrix.os == 'windows-latest'
        run: |
          (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python -
          echo "C:\\Users\\runneradmin\\AppData\\Roaming\\Python\\Scripts" >> $env:GITHUB_PATH

      - name: Verify Poetry Installation
        run: poetry --version

      - name: Clear Poetry Cache
        run: poetry cache clear pypi --all

      # Install dependencies, test, and remove for each extension
      - name: Install and test LLM extensions (Unix)
        if: matrix.os != 'windows-latest'
        run: |
          find extensions/llms -mindepth 1 -type d | while read -r dir; do
            if [ -f "$dir/pyproject.toml" ]; then
              echo "Installing dependencies for $dir"
              (
                cd "$dir" || exit
                poetry install --all-extras --with test --verbose
              )
              echo "Running tests for $dir"
              (
                cd "$dir" || exit
                poetry run pytest tests/
              )
              echo "Removing envs"
              (
                cd "$dir" || exit
                poetry env remove --all
              )
            fi
          done

      - name: Install and test Connector extensions (Unix)
        if: matrix.os != 'windows-latest'
        run: |
          find extensions/connectors -mindepth 1 -type d | while read -r dir; do
            if [ -f "$dir/pyproject.toml" ]; then
              echo "Installing dependencies for $dir"
              (
                cd "$dir" || exit
                poetry install --all-extras --with test --verbose
              )
              echo "Running tests for $dir"
              (
                cd "$dir" || exit
                poetry run pytest tests/
              )
              echo "Removing envs"
              (
                cd "$dir" || exit
                poetry env remove --all
              )
            fi
          done

      - name: Install and test Enterprise extensions (Unix)
        if: matrix.os != 'windows-latest'
        run: |
          find extensions/ee -mindepth 1 -type d | while read -r dir; do
            if [ -f "$dir/pyproject.toml" ]; then
              echo "Installing dependencies for $dir"
              (
                cd "$dir" || exit
                poetry install --all-extras --with test --verbose
              )
              echo "Running tests for $dir"
              (
                cd "$dir" || exit
                poetry run pytest tests/
              )
              echo "Removing envs"
              (
                cd "$dir" || exit
                poetry env remove --all
              )
            fi
          done

      - name: Run extension tests (Windows)
        if: matrix.os == 'windows-latest'
        run: |
          # Run LLM extension tests
          Get-ChildItem -Path extensions/llms -Directory | ForEach-Object {
            $testDir = Join-Path $_.FullName "tests"
            if (Test-Path $testDir) {
              Write-Host "Running tests for $($_.FullName)"
              Push-Location $_.FullName
              poetry install --all-extras --with test --verbose
              poetry run pytest tests/
              Pop-Location
            }
          }

          # Run connector extension tests
          Get-ChildItem -Path extensions/connectors -Directory | ForEach-Object {
            $testDir = Join-Path $_.FullName "tests"
            if (Test-Path $testDir) {
              Write-Host "Running tests for $($_.FullName)"
              Push-Location $_.FullName
              poetry install --all-extras --with test --verbose
              poetry run pytest tests/
              Pop-Location
            }
          }

          # Run enterprise extension tests
          Get-ChildItem -Path extensions/ee -Recurse -Directory -Depth 2 | ForEach-Object {
            $testDir = Join-Path $_.FullName "tests"
            if (Test-Path $testDir) {
              Write-Host "Running tests for $($_.FullName)"
              Push-Location $_.FullName
              poetry install --all-extras --with test --verbose
              Pop-Location
            }
          }

      - name: Run code coverage for extensions
        continue-on-error: true
        run: |
          pip install coverage
          poetry run coverage run --source=extensions -m pytest tests extensions/*/tests
          poetry run coverage xml

      - name: Report coverage
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: false