From 86dc039215e0a47d48dd06a9716b8b26385891bd Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 28 Feb 2024 06:40:53 +0200 Subject: [PATCH] [improve][ci] Run CodeQL within Pulsar CI workflow as mandatory check (#22132) --- .github/workflows/codeql.yaml | 10 ++-- .github/workflows/pulsar-ci-flaky.yaml | 2 + .github/workflows/pulsar-ci.yaml | 75 ++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 5fb3732c1de2b..6c7e9acfbf116 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -22,14 +22,18 @@ name: "CodeQL" on: push: branches: [ 'master' ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ 'master' ] schedule: - cron: '27 21 * * 4' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true jobs: analyze: + # only run scheduled analysis in apache/pulsar repository + if: ${{ (github.event_name == 'schedule' && github.repository == 'apache/pulsar') || github.event_name != 'schedule' }} name: Analyze runs-on: 'ubuntu-latest' timeout-minutes: 360 diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index 8403575e74cff..c8e944fcfc6d9 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -22,6 +22,8 @@ on: pull_request: branches: - master + - branch-* + - pulsar-* schedule: # scheduled job with JDK 17 - cron: '0 12 * * *' diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index effeab90beb95..feaf6d0a5a867 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -22,6 +22,8 @@ on: pull_request: branches: - master + - branch-* + - pulsar-* schedule: # scheduled job with JDK 17 - cron: '0 12 * * *' @@ -1331,6 +1333,71 @@ jobs: - name: build package run: mvn -B clean package -DskipTests -T 1C -ntp + codeql: + name: Run CodeQL Analysis + runs-on: ubuntu-22.04 + timeout-minutes: 60 + needs: ['preconditions', 'unit-tests'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }} + permissions: + actions: read + contents: read + security-events: write + env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + CODEQL_LANGUAGE: java-kotlin + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Clean Disk when needed + if: ${{ matrix.clean_disk }} + uses: ./.github/actions/clean-disk + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v4 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ env.CI_JDK_MAJOR_VERSION }} + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ env.CODEQL_LANGUAGE }} + + - name: Build Java code + run: | + mvn -B -ntp -Pcore-modules,-main install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{ env.CODEQL_LANGUAGE }}" + owasp-dep-check: name: OWASP dependency check runs-on: ubuntu-22.04 @@ -1429,7 +1496,7 @@ jobs: # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing. pulsar-ci-checks-completed: name: "Pulsar CI checks completed" - if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar')) }} + if: ${{ always() && needs.preconditions.result == 'success' }} runs-on: ubuntu-22.04 timeout-minutes: 10 needs: [ @@ -1442,7 +1509,8 @@ jobs: 'unit-tests-upload-coverage', 'integration-tests-upload-coverage', 'system-tests-upload-coverage', - 'owasp-dep-check' + 'owasp-dep-check', + 'codeql' ] steps: - name: Check that all required jobs were completed successfully @@ -1453,6 +1521,7 @@ jobs: && "${{ needs.integration-tests.result }}" == "success" \ && "${{ needs.system-tests.result }}" == "success" \ && "${{ needs.macos-build.result }}" == "success" \ + && ( "${{ needs.codeql.result }}" == "success" || "${{ needs.codeql.result }}" == "skipped" ) \ ) ]]; then echo "Required jobs haven't been completed successfully." exit 1 @@ -1474,4 +1543,4 @@ jobs: if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} run: | gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true - gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true + gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true \ No newline at end of file