From ec4f52ed54113a71febd3d71b70735ae94ebc633 Mon Sep 17 00:00:00 2001
From: Jon Simantov <jsimantov@google.com>
Date: Tue, 23 Jan 2024 10:40:22 -0800
Subject: [PATCH] Add retry to C++ packaging step.

---
 .github/workflows/cpp-packaging.yml | 32 +++++++++++++++++++++++++++++
 scripts/gha/retry_test_failures.py  | 15 ++++++++++++--
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml
index 91bc5d0112..32c26619fc 100644
--- a/.github/workflows/cpp-packaging.yml
+++ b/.github/workflows/cpp-packaging.yml
@@ -893,3 +893,35 @@ jobs:
             -s 10 \
             -A ${verbose_flag}
         fi
+
+
+  attempt_retry:
+    name: "attempt-retry"
+    needs: [trigger_integration_tests]
+    runs-on: ubuntu-20.04
+    if: ${{ failure() && !cancelled() && github.event_name == 'schedule' }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{needs.check_and_prepare.outputs.github_ref}}
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.pythonVersion }}
+      - name: Install python deps
+        run: pip install -r scripts/gha/python_requirements.txt
+      # The default token can't run workflows, so get an alternate token.
+      - name: Generate token for GitHub API
+        uses: tibdex/github-app-token@v1
+        id: generate-token
+        with:
+          app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
+          private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
+      - name: Retry failed tests
+        run: |
+          echo "::warning ::Attempting to retry failed tests"
+          python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} \
+          -w retry-test-failures.yml \
+          -p run_id ${{ github.run_id }} \
+          -s 10 \
+          -A
diff --git a/scripts/gha/retry_test_failures.py b/scripts/gha/retry_test_failures.py
index bb88cbe873..7aed13a1a1 100644
--- a/scripts/gha/retry_test_failures.py
+++ b/scripts/gha/retry_test_failures.py
@@ -107,13 +107,24 @@ def main(argv):
         # Retry all build failures that don't match a compiler error.
         if not re.search(r'.*/.*:[0-9]+:[0-9]+: error:', job_logs):
           should_rerun_jobs = True
+        # Also retry build jobs that timed out
+        if not re.search(r'exceeded the maximum execution time', job_logs):
+          should_rerun_jobs = True
+      elif job['name'].startswith('download-'):
+        # If a download step failed, automatically retry.
+          should_rerun_jobs = True
+      elif job['name'].startswith('package-'):
+        # Retry packaging jobs that timed out
+        if not re.search(r'exceeded the maximum execution time', job_logs):
+          should_rerun_jobs = True
       elif job['name'].startswith('test-'):
         if '-android' in job['name'] or '-ios' in job['name']:
           # Mobile tests should always be retried.
           should_rerun_jobs = True
         else:
-          # Desktop tests should only retry on a network error.
-          if re.search(r'timed out|network error', job_logs, re.IGNORECASE):
+          # Desktop tests should only retry on a network error or timeout.
+          if re.search(r'timed out|network error|maximum execution time',
+                       job_logs, re.IGNORECASE):
             should_rerun_jobs = True
 
   if should_rerun_jobs: