fix(CI_Pipeline): resolve dependency issues causing build failures #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci-extensions | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| jobs: | |
| extensions-tests: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macOS-latest] | |
| python-version: ["3.10", "3.11"] | |
| steps: | |
| - name: Clean up instance space | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| df -h | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Poetry (Unix) | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - | |
| echo 'export PATH="$HOME/.local/bin:$PATH"' >> $GITHUB_ENV | |
| - name: Install Poetry (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python - | |
| echo "C:\\Users\\runneradmin\\AppData\\Roaming\\Python\\Scripts" >> $env:GITHUB_PATH | |
| - name: Verify Poetry Installation | |
| run: poetry --version | |
| - name: Clear Poetry Cache | |
| run: poetry cache clear pypi --all | |
| # Install dependencies, test, and remove for each extension | |
| - name: Install and test LLM extensions (Unix) | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| find extensions/llms -mindepth 1 -type d | while read -r dir; do | |
| if [ -f "$dir/pyproject.toml" ]; then | |
| echo "Installing dependencies for $dir" | |
| ( | |
| cd "$dir" || exit | |
| poetry install --all-extras --with test --verbose | |
| ) | |
| echo "Running tests for $dir" | |
| ( | |
| cd "$dir" || exit | |
| poetry run pytest tests/ | |
| ) | |
| echo "Removing envs" | |
| ( | |
| cd "$dir" || exit | |
| poetry env list --all | awk '{print $1}' | xargs -n 1 poetry env remove | |
| ) | |
| fi | |
| done | |
| - name: Install and test Connector extensions (Unix) | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| find extensions/connectors -mindepth 1 -type d | while read -r dir; do | |
| if [ -f "$dir/pyproject.toml" ]; then | |
| echo "Installing dependencies for $dir" | |
| ( | |
| cd "$dir" || exit | |
| poetry install --all-extras --with test --verbose | |
| ) | |
| echo "Running tests for $dir" | |
| ( | |
| cd "$dir" || exit | |
| poetry run pytest tests/ | |
| ) | |
| echo "Removing envs" | |
| ( | |
| cd "$dir" || exit | |
| poetry env list --all | awk '{print $1}' | xargs -n 1 poetry env remove | |
| ) | |
| fi | |
| done | |
| - name: Install and test Enterprise extensions (Unix) | |
| if: matrix.os != 'windows-latest' | |
| run: | | |
| find extensions/ee -mindepth 1 -type d | while read -r dir; do | |
| if [ -f "$dir/pyproject.toml" ]; then | |
| echo "Installing dependencies for $dir" | |
| ( | |
| cd "$dir" || exit | |
| poetry install --all-extras --with test --verbose | |
| ) | |
| echo "Running tests for $dir" | |
| ( | |
| cd "$dir" || exit | |
| poetry run pytest tests/ | |
| ) | |
| echo "Removing envs" | |
| ( | |
| cd "$dir" || exit | |
| poetry env list --all | awk '{print $1}' | xargs -n 1 poetry env remove | |
| ) | |
| fi | |
| done | |
| - name: Run extension tests (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| # Run LLM extension tests | |
| Get-ChildItem -Path extensions/llms -Directory | ForEach-Object { | |
| $testDir = Join-Path $_.FullName "tests" | |
| if (Test-Path $testDir) { | |
| Write-Host "Running tests for $($_.FullName)" | |
| Push-Location $_.FullName | |
| poetry install --all-extras --with test --verbose | |
| poetry run pytest tests/ | |
| Pop-Location | |
| } | |
| } | |
| # Run connector extension tests | |
| Get-ChildItem -Path extensions/connectors -Directory | ForEach-Object { | |
| $testDir = Join-Path $_.FullName "tests" | |
| if (Test-Path $testDir) { | |
| Write-Host "Running tests for $($_.FullName)" | |
| Push-Location $_.FullName | |
| poetry install --all-extras --with test --verbose | |
| poetry run pytest tests/ | |
| Pop-Location | |
| } | |
| } | |
| # Run enterprise extension tests | |
| Get-ChildItem -Path extensions/ee -Recurse -Directory -Depth 2 | ForEach-Object { | |
| $testDir = Join-Path $_.FullName "tests" | |
| if (Test-Path $testDir) { | |
| Write-Host "Running tests for $($_.FullName)" | |
| Push-Location $_.FullName | |
| poetry install --all-extras --with test --verbose | |
| Pop-Location | |
| } | |
| } | |
| - name: Run code coverage for extensions | |
| continue-on-error: true | |
| run: | | |
| pip install coverage | |
| poetry run coverage run --source=extensions -m pytest tests extensions/*/tests | |
| poetry run coverage xml | |
| - name: Report coverage | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./coverage.xml | |
| flags: unittests | |
| name: codecov-umbrella | |
| fail_ci_if_error: false |