From b977942008be19e032c542df1dbbf73b19559525 Mon Sep 17 00:00:00 2001 From: Juraj Piar Date: Wed, 17 Apr 2024 00:53:58 +0100 Subject: [PATCH] feat(ci): migrates circleci to github actions feat(ci): adjusts mining-tests feat(ci): makes services work feat(ci): makes services work removes port verification trying docker compose inside mining tests trying docker compose inside mining tests trying docker compose inside mining tests trying docker compose inside mining tests using mining tests fork anew just bitcoin nodes build publish docker reatach rest of job node v -> 12 run tests in same step as rskj reenables build and other jobs no recompile in mining-tests feat(ci): uses rsksmart docker images only push ddocker image if different feat(ci): uses private ghcr --- .circleci/config.yml | 200 -------------------- .github/workflows/build_and_test.yml | 262 +++++++++++++++++++++++++++ build.gradle | 10 + 3 files changed, 272 insertions(+), 200 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/build_and_test.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 53bc7177463..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,200 +0,0 @@ -version: 2.1 # needed for executors - -executors: - rskj-executor: - docker: - - image: openjdk:8-jdk - environment: - _JAVA_OPTIONS: "-Xmx3G -Xms2G" - working_directory: /app - resource_class: medium+ - sonarqube-executor: - docker: - - image: eclipse-temurin:17-jdk - working_directory: /app - mit-executor: - docker: - - image: alpine:3.10 - -jobs: - build: - executor: rskj-executor - steps: - - checkout - - run: - name: Setup - command: apt update -y && apt install -y gnupg2 - - run: - name: Verify files - command: | - curl -sSL https://secchannel.rsk.co/SUPPORT.asc | gpg2 --import - - gpg2 --verify SHA256SUMS.asc && sha256sum --check SHA256SUMS.asc - - run: - name: Build - command: | - ./configure.sh - ./gradlew --no-daemon dependencies - ./gradlew --no-daemon --stacktrace build -x test - - persist_to_workspace: - root: . - paths: - - . - sonarqube: - executor: sonarqube-executor - steps: - - attach_workspace: - at: /app - - run: - name: Run SonarQube analysis - command: | - apt-get update && apt-get install -yqq git - extra_flags="" - if [ -n "$CIRCLE_PULL_REQUEST" ]; then - # https://community.sonarsource.com/t/no-code-or-issue-found-in-pull-request-decorations-github-circleci/8496 - git branch -f master origin/master - # extract PR number, as PR URLs are in the form - # https://github.com/$user/$repo/pull/$pr_number - pr_number=${CIRCLE_PULL_REQUEST##*/} - extra_flags="-Dsonar.pullrequest.base=master - -Dsonar.pullrequest.branch=$CIRCLE_BRANCH - -Dsonar.pullrequest.key=$pr_number" - else - extra_flags="-Dsonar.branch.name=master" - fi - ./gradlew sonarqube --no-daemon -x build -x test \ - $extra_flags \ - -Dsonar.organization=rsksmart \ - -Dsonar.host.url="$SONAR_URL" \ - -Dsonar.login="$SONAR_TOKEN" - rskj-unit-tests: - executor: rskj-executor - steps: - - attach_workspace: - at: /app - - run: - name: rskj tests - command: ./gradlew --no-daemon --stacktrace test - - run: - name: Save test results - command: | - mkdir -p ~/junit/ - find rskj-core/build/test-results -type f -name "*.xml" \ - -exec cp {} ~/junit/ \; - when: always - - store_test_results: - path: ~/junit - - store_artifacts: - path: ~/junit - - persist_to_workspace: - root: . - paths: - - . - rskj-int-tests: - executor: rskj-executor - steps: - - attach_workspace: - at: /app - - run: - name: rskj integration tests - command: ./gradlew --no-daemon --stacktrace integrationTest - - run: - name: Save test results - command: | - mkdir -p ~/junit/ - find rskj-core/build/test-results -type f -name "*.xml" \ - -exec cp {} ~/junit/ \; - when: always - - store_test_results: - path: ~/junit - - store_artifacts: - path: ~/junit - - persist_to_workspace: - root: . - paths: - - . - mining-tests: - executor: mit-executor - steps: - - run: - name: Setup - command: apk add --no-cache curl jq - - run: - name: Mining integration tests - command: | - wait_for_completion() { - # inspired from https://discuss.circleci.com/t/waiting-for-build-to-complete-when-invoked-via-http-api/14989 - build_number=$1 - poll_interval=60 - - i=0 - max_count=20 - while [ $i -lt $max_count ]; do - # output to avoid CircleCI considering the job stuck - res=$(curl -Ssfu "$CIRCLE_INTEGRATIONS_TOKENS:" \ - "https://circleci.com/api/v1.1/project/github/$MIT_ORGANIZATION/$MIT_PROJECT/$build_number" \ - | jq -r '[.lifecycle, .outcome] | @tsv') - IFS=" " set -- $res - lifecycle=${1:-} - outcome=${2:-} - if [ "$lifecycle" = "queued" ]; then - printf "Build is enqueued. Waiting...\n" - # don't increment $i - else - printf "[%02u/%02u] Waiting for build %s ...\n" \ - "$i" "$max_count" "$build_url" - i=$(($i + 1)) - fi - if [ "$lifecycle" = "finished" ]; then - printf "Build %u finished. Outcome: \"%s\".\n" \ - "$build_number" "$outcome" - # return success iff job outcome is "success" - test "$outcome" = "success" - return $? - fi - - sleep $poll_interval - done - return 1 - } - - json_payload='{ - "build_parameters": { - "RSKJ_CIRCLE_BRANCH": "'$CIRCLE_BRANCH'", - "RSKJ_CIRCLE_USERNAME": "'$CIRCLE_PROJECT_USERNAME'", - "RSKJ_CIRCLE_REPONAME": "'$CIRCLE_PROJECT_REPONAME'", - "RSKJ_PR_NUMBER": "'$CIRCLE_PR_NUMBER'", - "RSKJ_CIRCLE_SHA1": "'$CIRCLE_SHA1'" - } - }' - res=$(curl -Ssf -u "$CIRCLE_INTEGRATIONS_TOKEN:" \ - -H "Content-type: application/json" -d "$json_payload" \ - "https://circleci.com/api/v1.1/project/github/$MIT_ORGANIZATION/$MIT_PROJECT/tree/$MIT_BRANCH" \ - | jq -r '[.build_url, .build_num] | @tsv') - IFS=" " set -- $res - test $# -eq 2 # ensure exactly 2 values are expanded - build_url=$1 - build_num=$2 - printf "Running mining integration tests. Follow it on:\n\n %s\n\n" "$build_url" - sleep 10 # give CircleCI some time to spin up the job - wait_for_completion "$build_num" - -workflows: - # https://circleci.com/docs/2.0/configuration-reference/#version-1 - # As of commit date, it reads: - # Should currently be `2` - version: 2 - build-and-test: - jobs: - - build - - rskj-unit-tests: - requires: - - build - - rskj-int-tests: - requires: - - build - - sonarqube: - requires: - - rskj-unit-tests - - mining-tests: - requires: - - rskj-unit-tests diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 00000000000..421a05e5770 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,262 @@ +name: Build and Test + +on: + push: + branches: + - '**' + pull_request: + types: [opened, reopened] + branches: + - '**' + +jobs: + build: + runs-on: ubuntu-latest + container: + image: openjdk:8-jdk + env: + _JAVA_OPTIONS: "-Xmx3G -Xms2G" + defaults: + run: + working-directory: . + steps: + - uses: actions/checkout@v2 + - name: Setup System Tools + run: | + apt update -y + apt install -y gnupg2 curl + - name: Install Node.js + run: | + curl -fsSL https://deb.nodesource.com/setup_16.x | bash - + apt-get install -y nodejs + - name: Check Node.js version + run: node --version + - name: Verify files + run: | + curl -sSL https://secchannel.rsk.co/SUPPORT.asc | gpg2 --import - + gpg2 --verify SHA256SUMS.asc && sha256sum --check SHA256SUMS.asc + + - uses: actions/cache@v4 + name: Cache Gradle + with: + path: | + .gradle/caches + gradle/wrapper + rskj-core/build + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Build + run: | + ./configure.sh + ./gradlew --no-daemon dependencies + ./gradlew --no-daemon --stacktrace build -x test + + sonarqube: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Java JDK + uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + + - name: Verify Java installation + run: java -version + + - uses: actions/cache@v4 + name: Restore Gradle cache + with: + path: | + .gradle/caches + gradle/wrapper + rskj-core/build + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Get Gradle jar + run: | + if [ ! -f ./gradle/wrapper/gradle-wrapper.jar ]; then + ./configure.sh + fi + + - name: Run SonarQube analysis + run: | + sudo apt-get update && sudo apt-get install -yqq git + extra_flags="" + if [ -n "$CIRCLE_PULL_REQUEST" ]; then + # https://community.sonarsource.com/t/no-code-or-issue-found-in-pull-request-decorations-github-circleci/8496 + git branch -f master origin/master + # extract PR number, as PR URLs are in the form + # https://github.com/$user/$repo/pull/$pr_number + pr_number==${{ github.event.pull_request.number }} + extra_flags="-Dsonar.pullrequest.base=master + -Dsonar.pullrequest.branch=${{ github.head_ref }} + -Dsonar.pullrequest.key=$pr_number" + else + extra_flags="-Dsonar.branch.name=master" + fi + ./gradlew sonarqube --no-daemon -x build -x test \ + $extra_flags \ + -Dsonar.organization=rsksmart \ + -Dsonar.host.url="https://sonarcloud.io" \ + -Dsonar.login="${{ secrets.SONAR_TOKEN }}" + + tests: + needs: build + runs-on: ubuntu-latest + container: + image: openjdk:8-jdk + defaults: + run: + working-directory: . + steps: + - uses: actions/checkout@v2 + + - name: Setup System Tools + run: | + apt update -y + apt install -y curl + - name: Install Node.js + run: | + curl -fsSL https://deb.nodesource.com/setup_16.x | bash - + apt-get install -y nodejs + - name: Check Node.js version + run: node --version + + - uses: actions/cache@v4 + name: Restore gradle cache + with: + path: | + .gradle/caches + gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Get Gradle jar + run: | + if [ ! -f ./gradle/wrapper/gradle-wrapper.jar ]; then + ./configure.sh + fi + + - name: Unit tests + run: | + chmod +x ./gradlew + ./gradlew --no-daemon --stacktrace test + - name: Integration tests + run: | + chmod +x ./gradlew + ./gradlew --no-daemon --stacktrace integrationTest + + - name: Save test results + run: | + mkdir -p ~/junit/ + find rskj-core/build/test-results -type f -name "*.xml" -exec cp {} ~/junit/ \; + if: always() + + mining-tests: + needs: build + runs-on: ubuntu-latest + services: + bitcoind1: + image: ghcr.io/rsksmart/rskj/mit_bitcoind1:latest + ports: + - 8331:8331 + - 31591:31591 + - 32591:32591 + options: --name bitcoind1 + bitcoind2: + image: ghcr.io/rsksmart/rskj/mit_bitcoind2:latest + ports: + - 8332:8332 + - 31592:31592 + - 32592:32592 + options: --name bitcoind2 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '12.x' + - name: Check Node.js version + run: node --version + + - name: Checkout Mining Integration Tests Repository + uses: actions/checkout@v4 + with: + repository: rsksmart/mining-integration-tests + ref: ${{ secrets.MINING_INTEGRATION_TESTS_REF }} + token: ${{ secrets.GITHUB_TOKEN }} + path: mining-integration-tests + + - name: Install Mining Integration Tests dependencies + working-directory: mining-integration-tests + run: | + npm ci + + - name: Change mining-integration-tests bitcoind url in config.json to localhost + working-directory: mining-integration-tests + run: | + sed -i 's/bitcoind01/localhost/g' config.json + + - name: Generate BTC blocks + working-directory: mining-integration-tests + run: | + node --unhandled-rejections=strict generateBtcBlocks.js + + - name: Setup Java JDK + uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + + - name: Verify Java installation + run: java -version + + - uses: actions/cache@v4 + name: Restore gradle cache + with: + path: | + .gradle/caches + gradle/wrapper + rskj-core/build + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Get Gradle jar + run: | + if [ ! -f ./gradle/wrapper/gradle-wrapper.jar ]; then + ./configure.sh + fi + + - name: Start RSKj and Run Tests + working-directory: mining-integration-tests + run: | + version=$(tr -d "'\"" < ../rskj-core/src/main/resources/version.properties \ + | cut -d = -f 2- | paste -sd - -) + ls -la ../rskj-core/build/libs/rskj-core-"$version"-all.jar + java -Drsk.conf.file=./rsk-integration-test.conf -cp ../rskj-core/build/libs/rskj-core-"$version"-all.jar co.rsk.Start --regtest & rskpid=$! + until nc -z 127.0.0.1 4444 + do + echo "Waiting for RskJ..." + sleep 1 + done + npm test + kill $rskpid + + - name: Review test results + working-directory: mining-integration-tests + if: always() + run: | + cat ./Results/mocha/Test-Results.xml diff --git a/build.gradle b/build.gradle index e26d0acfee6..b6d9f595e47 100644 --- a/build.gradle +++ b/build.gradle @@ -7,3 +7,13 @@ subprojects { group = 'co.rsk' version = config.modifier?.trim() ? config.versionNumber + "-" + config.modifier : config.versionNumber } + +sonarqube { + properties { + property "sonar.java.binaries", "rskj-core/build/classes/java/main" + property "sonar.scm.provider", "git" + property "sonar.projectBaseDir", project.projectDir + property "sonar.projectKey", "rskj" + property "sonar.organization", "rsksmart" + } +}