test: CI: complete dockers_builder.yml

hove-io · Sep 30, 2024 · 9d317a8 · 9d317a8
1 parent 4b9c475
commit 9d317a8
Show file tree

Hide file tree

Showing 5 changed files with 157 additions and 34 deletions.
diff --git a/.github/workflows/build_dockers_debian11.yml b/.github/workflows/build_dockers_debian11.yml
@@ -10,7 +10,6 @@ env:
   debian_version: debian11
 
 jobs:
-
   build_navitia_arm64:
     runs-on: ubuntu-latest
     steps:

diff --git a/...orkflows/docker_build_kraken_debian11.yml → ...kflows/docker_builder_kraken_debian11.yml b/...orkflows/docker_build_kraken_debian11.yml → ...kflows/docker_builder_kraken_debian11.yml
@@ -9,7 +9,6 @@ on:
       - '*'
 
 jobs:
-
   common_variables:
     name: Common variables
     runs-on: [self-hosted, kraken, sandbox]

diff --git a/.github/workflows/dockers_builder.yml b/.github/workflows/dockers_builder.yml
@@ -8,8 +8,9 @@ on:
       - '*'
 
 env:
-  front_debian8_components: 'tyr-web instances-configurator'
-  backend_debian8_components: 'kraken tyr-beat tyr-worker mock-kraken eitri'
+  front_components: 'tyr-web instances-configurator'
+  backend_components: 'kraken tyr-beat tyr-worker mock-kraken eitri'
+  backend_core_components: 'jormungandr kraken tyr-beat tyr-worker'
 
   SBX_ECR_REGISTRY_BACKEND: "110444322584.dkr.ecr.eu-west-1.amazonaws.com"
   SBX_ECR_REGISTRY_FRONT: "051314639660.dkr.ecr.eu-west-1.amazonaws.com"
@@ -24,38 +25,34 @@ jobs:
     steps:
       - name: force chown to avoid errors
         run: sudo chown -R $USER:$USER .
-
       - name: Generate github private access token
         id: ci-core-app-token
         uses: getsentry/[email protected]
         with:
           app_id: ${{ secrets.CI_CORE_APP_ID }}
           private_key: ${{ secrets.CI_CORE_APP_PEM }}
-
       - name: Checkout navitia
         id: checkout_navitia
         uses: actions/checkout@v3
         with:
           token: ${{ steps.ci-core-app-token.outputs.token }}
           # we need entire history for tags
           fetch-depth: 0
-
       - name: Choose navitia tag
         id: choose_navitia_tag
         run: |
           version=$(git describe --tags)
           echo "navitia_tag=$version" >> $GITHUB_OUTPUT
-
       - name: failure notification
         if: failure()
         run: |
           sudo apt update && sudo apt install -y httpie
           echo '{"text":":warning: Github Actions: workflow dockers_builder common_variables failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}}
 
 
-  debian11_images:
+  debian11_back_images:
     runs-on: [self-hosted, corefront, sandbox]
-    name: Build debian11 images
+    name: Build debian11 backend images
     needs: common_variables
     steps:
       - name: force chown to avoid errors
@@ -84,35 +81,49 @@ jobs:
         with:
           # 162230498103 : shared
           # 051314639660 : corefront sbx
-          registries: "162230498103,051314639660"
+          # 110444322584 : kraken sbx
+          registries: "162230498103,110444322584,051314639660"
 
-      - name: Create builder docker
+      - name: Create master&builder docker
         run: |
+          docker build -f docker/debian11/Dockerfile-master -t navitia/master_debian11 .
           docker build -f docker/debian11/Dockerfile-builder -t navitia/builder_debian11 .
 
-      - name: Build navitia
+      - name: Build packages in master docker
         run: |
           docker run -v `pwd`:/navitia/navitia/  navitia/builder_debian11
 
       - name: Create navitia images
-        run: |
-          echo "*********  Building Jormungandr ***************"
-          docker build -t navitia/jormungandr_debian11 -f  docker/debian11/Dockerfile-jormungandr .
+        run:  |
+          for component in ${{env.backend_core_components}}; do
+              echo "*********  Building $component ***************"
+              docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f  docker/debian11/Dockerfile-${component} .
+          done
 
-      - name: Push jormun dev image on SBX ECR
+      - name: Push dev images on SBX ECR
         if: github.ref == 'refs/heads/dev'
         run: |
           jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_dev
           docker tag navitia/jormungandr_debian11 ${jormun_tag}
           docker push ${jormun_tag}
 
-      - name: Push jormun release image on SBX ECR
+      - name: Push release images on PRD ECR
         if: startsWith(github.ref, 'refs/tags/')
         run: |
-          jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${{ needs.common_variables.outputs.RELEASE_TAG }}
-          docker tag navitia/jormungandr_debian11 ${jormun_tag}
-          docker push ${jormun_tag}
+          # Kraken
+          kraken_tag=${PRD_ECR_REGISTRY}/navitia-debian11-kraken-kraken:${{ needs.common_variables.outputs.RELEASE_TAG }}
+          docker tag navitia/kraken ${kraken_tag}
+          docker push ${kraken_tag}
+
+          # Tyr-beat
+          tyr_beat_tag=${PRD_ECR_REGISTRY}/navitia-debian11-tyr-tyrbeat:${{ needs.common_variables.outputs.RELEASE_TAG }}
+          docker tag navitia/tyr-beat ${tyr_beat_tag}
+          docker push ${tyr_beat_tag}
 
+          # Tyr-worker
+          tyr_worker_tag=${PRD_ECR_REGISTRY}/navitia-debian11-tyr-tyrworker:${{ needs.common_variables.outputs.RELEASE_TAG }}
+          docker tag navitia/tyr-worker ${tyr_worker_tag}
+          docker push ${tyr_worker_tag}
 
       - name: failure notification
         if: failure()
@@ -127,6 +138,88 @@ jobs:
           rm -rf ./*
           rm -rf ./.??*
 
+
+  debian11_front_images:
+    runs-on: [self-hosted, corefront, sandbox]
+    name: Build debian11 front images
+    needs: common_variables
+    steps:
+      - name: force chown to avoid errors
+        run: sudo chown -R $USER:$USER .
+
+      - name: Git config
+        run: git config --global --add safe.directory /__w/navitia/navitia
+
+      - name: Generate github private access token
+        id: ci-core-app-token
+        uses: getsentry/[email protected]
+        with:
+          app_id: ${{ secrets.CI_CORE_APP_ID }}
+          private_key: ${{ secrets.CI_CORE_APP_PEM }}
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          token: ${{ steps.ci-core-app-token.outputs.token }}
+
+      - name: Login to Amazon ECR
+        uses: aws-actions/amazon-ecr-login@v1
+        env:
+          AWS_REGION: eu-west-1
+        with:
+          # 162230498103 : shared
+          # 051314639660 : corefront sbx
+          # 110444322584 : kraken sbx
+          registries: "162230498103,110444322584,051314639660"
+
+      - name: Create master&builder docker
+        run: |
+          docker build -f docker/debian11/Dockerfile-master -t navitia/master_debian11 .
+          docker build -f docker/debian11/Dockerfile-builder -t navitia/builder_debian11 .
+
+      - name: Build packages in master docker
+        run: |
+          docker run -v `pwd`:/navitia/navitia/  navitia/builder_debian11
+
+      - name: Create navitia images
+        run:  |
+          for component in ${{env.front_components}}; do
+              echo "*********  Building $component ***************"
+              docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f  docker/debian11/Dockerfile-${component} .
+          done
+
+      - name: Push dev images on SBX ECR
+        if: github.ref == 'refs/heads/dev'
+        run: |
+          for component in ${{env.front_components}}; do
+              component_tag=${SBX_ECR_REGISTRY_FRONT}/${component}:dev
+              docker tag navitia/$component ${component_tag}
+              docker push ${component_tag}
+          done
+
+      - name: Push release images on PRD ECR
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          # Tyr-web
+          tyr_web_tag=${PRD_ECR_REGISTRY}/navitia-tyr-web-tyr-web:${{ needs.common_variables.outputs.RELEASE_TAG }}
+          docker tag navitia/tyr-web ${tyr_web_tag}
+          docker push ${tyr_web_tag}
+
+      - name: failure notification
+        if: failure()
+        run: |
+          echo '{"text":":warning: Github Actions: workflow dockers_builder debian11_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}}
+
+      - name: clean up workspace
+        if: ${{ always() }}
+        run: |
+          # some files are created by a docker container
+          sudo chown -R $USER:$USER .
+          rm -rf ./*
+          rm -rf ./.??*
+
+
   debian8_back_images:
     runs-on: [self-hosted, kraken, sandbox]
     name: Build debian8 back images
@@ -198,15 +291,15 @@ jobs:
 
       - name: Create navitia images
         run: |
-          for component in ${{env.backend_debian8_components}}; do
+          for component in ${{env.backend_components}}; do
               echo "*********  Building $component ***************"
               docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f  docker/debian8/Dockerfile-${component} .
           done
 
       - name: Push dev images on SBX ECR
         if: github.ref == 'refs/heads/dev'
         run: |
-          for component in ${{env.backend_debian8_components}}; do
+          for component in ${{env.backend_components}}; do
             component_tag=${SBX_ECR_REGISTRY_BACKEND}/${component}:dev
             docker tag navitia/$component ${component_tag}
             docker push ${component_tag}
@@ -288,15 +381,15 @@ jobs:
 
       - name: Create navitia images
         run: |
-          for component in ${{env.front_debian8_components}}; do
+          for component in ${{env.front_components}}; do
               echo "*********  Building $component ***************"
               docker build -t navitia/$component -f  docker/debian8/Dockerfile-${component} .
           done
 
       - name: Push dev images on SBX ECR
         if: github.ref == 'refs/heads/dev'
         run: |
-          for component in ${{env.front_debian8_components}}; do
+          for component in ${{env.front_components}}; do
               component_tag=${SBX_ECR_REGISTRY_FRONT}/${component}:dev
               docker tag navitia/$component ${component_tag}
               docker push ${component_tag}
@@ -368,7 +461,6 @@ jobs:
     name: Run artemis Dispatch (Dev)
     if: github.ref == 'refs/heads/dev'
     steps:
-
       - name: Generate github private access token
         id: ci-core-app-token
         uses: getsentry/[email protected]

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -13,7 +13,6 @@ env:
   REGION: eu-west-1
 
 jobs:
-
   credentials:
     name: Init credentials
     runs-on: [self-hosted, corefront, sandbox]
@@ -118,19 +117,16 @@ jobs:
         fail-fast: false
         matrix:
             os: [{docker_image: mutable-debian8_dev , python_version: python3.6}, {docker_image: mutable-debian11_dev, python_version: python3.9}]
-
     container:
         image: 162230498103.dkr.ecr.eu-west-1.amazonaws.com/${{ matrix.os.docker_image }}:latest
         credentials:
           username: AWS
           password: ${{ needs.credentials.outputs.aws_token }}
-
     services:
       rabbitmq:
         image: rabbitmq:3-alpine
         ports:
           - 5672:5672
-
       redis:
         image: redis:6-alpine
         ports:
@@ -146,7 +142,6 @@ jobs:
       with:
         app_id: ${{ secrets.CI_CORE_APP_ID }}
         private_key: ${{ secrets.CI_CORE_APP_PEM }}
-
     - uses: actions/checkout@v3
       with:
         submodules: recursive
@@ -200,7 +195,6 @@ jobs:
         NAVITIA_DOCKER_NETWORK: ${{ job.container.network }}
         TYR_CELERY_BROKER_URL: 'amqp://guest:guest@rabbitmq:5672//'
         TYR_REDIS_HOST: 'redis'
-
     - name: docker_test python3
       working-directory: ./build
       run: |
@@ -212,12 +206,10 @@ jobs:
         make docker_test
         deactivate
         rm -rf navitia_py3
-
       env:
         NAVITIA_DOCKER_NETWORK: ${{ job.container.network }}
         TYR_CELERY_BROKER_URL: 'amqp://guest:guest@rabbitmq:5672//'
         TYR_REDIS_HOST: 'redis'
-
     - name: clean up workspace
       if: ${{ always() }}
       run: |

diff --git a/docker/debian11/Dockerfile-master b/docker/debian11/Dockerfile-master
@@ -0,0 +1,41 @@
+FROM 162230498103.dkr.ecr.eu-west-1.amazonaws.com/mutable-debian11_dev:latest
+
+RUN echo "deb http://archive.debian.org/debian/ bullseye main" > /etc/apt/sources.list.d/archive.list
+
+# update package list from providers
+RUN apt-get update --force-yes --fix-missing || exit 0
+
+# install postgresql-client for tyr-beat
+#         netcat for kraken
+#         curl for jormun
+RUN apt-get install -y --force-yes ca-certificates \
+        postgresql-client \
+        netcat \
+        dh-python \
+        curl
+
+COPY docker/ca-certificates/*.crt /usr/local/share/ca-certificates/
+RUN update-ca-certificates
+
+# install a more recent version of CMake
+RUN apt-get remove --yes cmake \
+    && cd /tmp \
+    && wget https://cmake.org/files/v3.18/cmake-3.18.6-Linux-x86_64.tar.gz \
+    && tar xf cmake-3.18.6-Linux-x86_64.tar.gz --strip-components=1 --directory=/usr/local \
+    && rm -f cmake-3.18.6-Linux-x86_64.tar.gz \
+    && cd -
+
+# Python 'requests' package handle its own CA certificate list
+# Let's force it to use the OS's list
+ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
+
+# install rustup
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo
+
+ENV PATH=/usr/local/cargo/bin:${PATH}
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal \
+    && chmod -R a+wrX ${RUSTUP_HOME} ${CARGO_HOME} \
+    && rustup --version \
+    && cargo --version \
+    && rustc --version
-Original file line number
+Diff line change
@@ Expand Up / @@ -10,7 +10,6 @@ env: @@
       debian_version: debian11
     jobs:
       build_navitia_arm64:
         runs-on: ubuntu-latest
         steps:
@@ Expand Down @@