From 9d317a8790736d17a92e5f7027d438d7b2e24846 Mon Sep 17 00:00:00 2001 From: Adrien Chapelet Date: Mon, 30 Sep 2024 15:16:50 +0200 Subject: [PATCH] test: CI: complete dockers_builder.yml --- .github/workflows/build_dockers_debian11.yml | 1 - ...yml => docker_builder_kraken_debian11.yml} | 1 - .github/workflows/dockers_builder.yml | 140 +++++++++++++++--- .github/workflows/workflow.yml | 8 - docker/debian11/Dockerfile-master | 41 +++++ 5 files changed, 157 insertions(+), 34 deletions(-) rename .github/workflows/{docker_build_kraken_debian11.yml => docker_builder_kraken_debian11.yml} (99%) create mode 100644 docker/debian11/Dockerfile-master diff --git a/.github/workflows/build_dockers_debian11.yml b/.github/workflows/build_dockers_debian11.yml index 2508471bac..14e492d9cf 100644 --- a/.github/workflows/build_dockers_debian11.yml +++ b/.github/workflows/build_dockers_debian11.yml @@ -10,7 +10,6 @@ env: debian_version: debian11 jobs: - build_navitia_arm64: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/docker_build_kraken_debian11.yml b/.github/workflows/docker_builder_kraken_debian11.yml similarity index 99% rename from .github/workflows/docker_build_kraken_debian11.yml rename to .github/workflows/docker_builder_kraken_debian11.yml index f77fbf6bba..1bfc70df81 100644 --- a/.github/workflows/docker_build_kraken_debian11.yml +++ b/.github/workflows/docker_builder_kraken_debian11.yml @@ -9,7 +9,6 @@ on: - '*' jobs: - common_variables: name: Common variables runs-on: [self-hosted, kraken, sandbox] diff --git a/.github/workflows/dockers_builder.yml b/.github/workflows/dockers_builder.yml index 976c29fc46..b40a259d69 100644 --- a/.github/workflows/dockers_builder.yml +++ b/.github/workflows/dockers_builder.yml @@ -8,8 +8,9 @@ on: - '*' env: - front_debian8_components: 'tyr-web instances-configurator' - backend_debian8_components: 'kraken tyr-beat tyr-worker mock-kraken eitri' + front_components: 'tyr-web instances-configurator' + backend_components: 'kraken tyr-beat tyr-worker mock-kraken eitri' + backend_core_components: 'jormungandr kraken tyr-beat tyr-worker' SBX_ECR_REGISTRY_BACKEND: "110444322584.dkr.ecr.eu-west-1.amazonaws.com" SBX_ECR_REGISTRY_FRONT: "051314639660.dkr.ecr.eu-west-1.amazonaws.com" @@ -24,14 +25,12 @@ jobs: steps: - name: force chown to avoid errors run: sudo chown -R $USER:$USER . - - name: Generate github private access token id: ci-core-app-token uses: getsentry/action-github-app-token@v2.0.0 with: app_id: ${{ secrets.CI_CORE_APP_ID }} private_key: ${{ secrets.CI_CORE_APP_PEM }} - - name: Checkout navitia id: checkout_navitia uses: actions/checkout@v3 @@ -39,13 +38,11 @@ jobs: token: ${{ steps.ci-core-app-token.outputs.token }} # we need entire history for tags fetch-depth: 0 - - name: Choose navitia tag id: choose_navitia_tag run: | version=$(git describe --tags) echo "navitia_tag=$version" >> $GITHUB_OUTPUT - - name: failure notification if: failure() run: | @@ -53,9 +50,9 @@ jobs: echo '{"text":":warning: Github Actions: workflow dockers_builder common_variables failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - debian11_images: + debian11_back_images: runs-on: [self-hosted, corefront, sandbox] - name: Build debian11 images + name: Build debian11 backend images needs: common_variables steps: - name: force chown to avoid errors @@ -84,35 +81,49 @@ jobs: with: # 162230498103 : shared # 051314639660 : corefront sbx - registries: "162230498103,051314639660" + # 110444322584 : kraken sbx + registries: "162230498103,110444322584,051314639660" - - name: Create builder docker + - name: Create master&builder docker run: | + docker build -f docker/debian11/Dockerfile-master -t navitia/master_debian11 . docker build -f docker/debian11/Dockerfile-builder -t navitia/builder_debian11 . - - name: Build navitia + - name: Build packages in master docker run: | docker run -v `pwd`:/navitia/navitia/ navitia/builder_debian11 - name: Create navitia images - run: | - echo "********* Building Jormungandr ***************" - docker build -t navitia/jormungandr_debian11 -f docker/debian11/Dockerfile-jormungandr . + run: | + for component in ${{env.backend_core_components}}; do + echo "********* Building $component ***************" + docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f docker/debian11/Dockerfile-${component} . + done - - name: Push jormun dev image on SBX ECR + - name: Push dev images on SBX ECR if: github.ref == 'refs/heads/dev' run: | jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_dev docker tag navitia/jormungandr_debian11 ${jormun_tag} docker push ${jormun_tag} - - name: Push jormun release image on SBX ECR + - name: Push release images on PRD ECR if: startsWith(github.ref, 'refs/tags/') run: | - jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${{ needs.common_variables.outputs.RELEASE_TAG }} - docker tag navitia/jormungandr_debian11 ${jormun_tag} - docker push ${jormun_tag} + # Kraken + kraken_tag=${PRD_ECR_REGISTRY}/navitia-debian11-kraken-kraken:${{ needs.common_variables.outputs.RELEASE_TAG }} + docker tag navitia/kraken ${kraken_tag} + docker push ${kraken_tag} + + # Tyr-beat + tyr_beat_tag=${PRD_ECR_REGISTRY}/navitia-debian11-tyr-tyrbeat:${{ needs.common_variables.outputs.RELEASE_TAG }} + docker tag navitia/tyr-beat ${tyr_beat_tag} + docker push ${tyr_beat_tag} + # Tyr-worker + tyr_worker_tag=${PRD_ECR_REGISTRY}/navitia-debian11-tyr-tyrworker:${{ needs.common_variables.outputs.RELEASE_TAG }} + docker tag navitia/tyr-worker ${tyr_worker_tag} + docker push ${tyr_worker_tag} - name: failure notification if: failure() @@ -127,6 +138,88 @@ jobs: rm -rf ./* rm -rf ./.??* + + debian11_front_images: + runs-on: [self-hosted, corefront, sandbox] + name: Build debian11 front images + needs: common_variables + steps: + - name: force chown to avoid errors + run: sudo chown -R $USER:$USER . + + - name: Git config + run: git config --global --add safe.directory /__w/navitia/navitia + + - name: Generate github private access token + id: ci-core-app-token + uses: getsentry/action-github-app-token@v2.0.0 + with: + app_id: ${{ secrets.CI_CORE_APP_ID }} + private_key: ${{ secrets.CI_CORE_APP_PEM }} + + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ steps.ci-core-app-token.outputs.token }} + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v1 + env: + AWS_REGION: eu-west-1 + with: + # 162230498103 : shared + # 051314639660 : corefront sbx + # 110444322584 : kraken sbx + registries: "162230498103,110444322584,051314639660" + + - name: Create master&builder docker + run: | + docker build -f docker/debian11/Dockerfile-master -t navitia/master_debian11 . + docker build -f docker/debian11/Dockerfile-builder -t navitia/builder_debian11 . + + - name: Build packages in master docker + run: | + docker run -v `pwd`:/navitia/navitia/ navitia/builder_debian11 + + - name: Create navitia images + run: | + for component in ${{env.front_components}}; do + echo "********* Building $component ***************" + docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f docker/debian11/Dockerfile-${component} . + done + + - name: Push dev images on SBX ECR + if: github.ref == 'refs/heads/dev' + run: | + for component in ${{env.front_components}}; do + component_tag=${SBX_ECR_REGISTRY_FRONT}/${component}:dev + docker tag navitia/$component ${component_tag} + docker push ${component_tag} + done + + - name: Push release images on PRD ECR + if: startsWith(github.ref, 'refs/tags/') + run: | + # Tyr-web + tyr_web_tag=${PRD_ECR_REGISTRY}/navitia-tyr-web-tyr-web:${{ needs.common_variables.outputs.RELEASE_TAG }} + docker tag navitia/tyr-web ${tyr_web_tag} + docker push ${tyr_web_tag} + + - name: failure notification + if: failure() + run: | + echo '{"text":":warning: Github Actions: workflow dockers_builder debian11_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} + + - name: clean up workspace + if: ${{ always() }} + run: | + # some files are created by a docker container + sudo chown -R $USER:$USER . + rm -rf ./* + rm -rf ./.??* + + debian8_back_images: runs-on: [self-hosted, kraken, sandbox] name: Build debian8 back images @@ -198,7 +291,7 @@ jobs: - name: Create navitia images run: | - for component in ${{env.backend_debian8_components}}; do + for component in ${{env.backend_components}}; do echo "********* Building $component ***************" docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f docker/debian8/Dockerfile-${component} . done @@ -206,7 +299,7 @@ jobs: - name: Push dev images on SBX ECR if: github.ref == 'refs/heads/dev' run: | - for component in ${{env.backend_debian8_components}}; do + for component in ${{env.backend_components}}; do component_tag=${SBX_ECR_REGISTRY_BACKEND}/${component}:dev docker tag navitia/$component ${component_tag} docker push ${component_tag} @@ -288,7 +381,7 @@ jobs: - name: Create navitia images run: | - for component in ${{env.front_debian8_components}}; do + for component in ${{env.front_components}}; do echo "********* Building $component ***************" docker build -t navitia/$component -f docker/debian8/Dockerfile-${component} . done @@ -296,7 +389,7 @@ jobs: - name: Push dev images on SBX ECR if: github.ref == 'refs/heads/dev' run: | - for component in ${{env.front_debian8_components}}; do + for component in ${{env.front_components}}; do component_tag=${SBX_ECR_REGISTRY_FRONT}/${component}:dev docker tag navitia/$component ${component_tag} docker push ${component_tag} @@ -368,7 +461,6 @@ jobs: name: Run artemis Dispatch (Dev) if: github.ref == 'refs/heads/dev' steps: - - name: Generate github private access token id: ci-core-app-token uses: getsentry/action-github-app-token@v2.0.0 diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index fc5013464a..6fbe971ce0 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -13,7 +13,6 @@ env: REGION: eu-west-1 jobs: - credentials: name: Init credentials runs-on: [self-hosted, corefront, sandbox] @@ -118,19 +117,16 @@ jobs: fail-fast: false matrix: os: [{docker_image: mutable-debian8_dev , python_version: python3.6}, {docker_image: mutable-debian11_dev, python_version: python3.9}] - container: image: 162230498103.dkr.ecr.eu-west-1.amazonaws.com/${{ matrix.os.docker_image }}:latest credentials: username: AWS password: ${{ needs.credentials.outputs.aws_token }} - services: rabbitmq: image: rabbitmq:3-alpine ports: - 5672:5672 - redis: image: redis:6-alpine ports: @@ -146,7 +142,6 @@ jobs: with: app_id: ${{ secrets.CI_CORE_APP_ID }} private_key: ${{ secrets.CI_CORE_APP_PEM }} - - uses: actions/checkout@v3 with: submodules: recursive @@ -200,7 +195,6 @@ jobs: NAVITIA_DOCKER_NETWORK: ${{ job.container.network }} TYR_CELERY_BROKER_URL: 'amqp://guest:guest@rabbitmq:5672//' TYR_REDIS_HOST: 'redis' - - name: docker_test python3 working-directory: ./build run: | @@ -212,12 +206,10 @@ jobs: make docker_test deactivate rm -rf navitia_py3 - env: NAVITIA_DOCKER_NETWORK: ${{ job.container.network }} TYR_CELERY_BROKER_URL: 'amqp://guest:guest@rabbitmq:5672//' TYR_REDIS_HOST: 'redis' - - name: clean up workspace if: ${{ always() }} run: | diff --git a/docker/debian11/Dockerfile-master b/docker/debian11/Dockerfile-master new file mode 100644 index 0000000000..cb50d5735b --- /dev/null +++ b/docker/debian11/Dockerfile-master @@ -0,0 +1,41 @@ +FROM 162230498103.dkr.ecr.eu-west-1.amazonaws.com/mutable-debian11_dev:latest + +RUN echo "deb http://archive.debian.org/debian/ bullseye main" > /etc/apt/sources.list.d/archive.list + +# update package list from providers +RUN apt-get update --force-yes --fix-missing || exit 0 + +# install postgresql-client for tyr-beat +# netcat for kraken +# curl for jormun +RUN apt-get install -y --force-yes ca-certificates \ + postgresql-client \ + netcat \ + dh-python \ + curl + +COPY docker/ca-certificates/*.crt /usr/local/share/ca-certificates/ +RUN update-ca-certificates + +# install a more recent version of CMake +RUN apt-get remove --yes cmake \ + && cd /tmp \ + && wget https://cmake.org/files/v3.18/cmake-3.18.6-Linux-x86_64.tar.gz \ + && tar xf cmake-3.18.6-Linux-x86_64.tar.gz --strip-components=1 --directory=/usr/local \ + && rm -f cmake-3.18.6-Linux-x86_64.tar.gz \ + && cd - + +# Python 'requests' package handle its own CA certificate list +# Let's force it to use the OS's list +ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt + +# install rustup +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo + +ENV PATH=/usr/local/cargo/bin:${PATH} +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal \ + && chmod -R a+wrX ${RUSTUP_HOME} ${CARGO_HOME} \ + && rustup --version \ + && cargo --version \ + && rustc --version