eval-py.yml

name: Run Python evals

on:
  push:
    # files:
    #   - 'test-eval/**'

permissions:
  pull-requests: write
  contents: read

jobs:
  eval:
    name: Run Python evals
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: "recursive"

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.12" # TODO: Matrix test different versions

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r test-eval-py/requirements.txt

      - name: Run Evals
        uses: ./
        with:
          api_key: ${{ secrets.BRAINTRUST_API_KEY }}
          root: test-eval-py
          runtime: python # Assuming the action supports a 'python' runtime


      # - name: Start terminal session
      #   uses: mxschmitt/action-tmate@v3
      #   with:
      #     limit-access-to-actor: true