From 4f007809252e1b4747da0c2c0917ba7c85a184c0 Mon Sep 17 00:00:00 2001 From: Salaton Date: Mon, 15 Apr 2024 12:10:22 +0300 Subject: [PATCH] fix: add db connection details to celery deployments --- .github/workflows/testing.yaml | 7 ++- .../templates/celery/ocl_celery_beat.yaml | 4 +- .../celery/ocl_celery_bulk_import.yaml | 41 +++++++++++++- .../celery/ocl_celery_bulk_import_0_1.yaml | 41 +++++++++++++- .../celery/ocl_celery_bulk_import_2_3.yaml | 42 ++++++++++++++- .../celery/ocl_celery_concurrent.yaml | 41 +++++++++++++- .../templates/celery/ocl_celery_worker.yaml | 41 +++++++++++++- .../celery/ocl_celery_worker_indexing.yaml | 53 ++++++++++++++++++- .../templates/oclapi/api_deployment.yaml | 9 ++-- deploy/charts/openconceptlab/values.yaml | 4 +- deploy/deploy.sh | 6 ++- 11 files changed, 272 insertions(+), 17 deletions(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 17777d0..aac37ee 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -14,11 +14,14 @@ env: APP_NAME: ${{ secrets.APP_NAME }} REDIS_HOST: ${{ secrets.REDIS_HOST }} REDIS_PORT: ${{ secrets.REDIS_PORT }} + API_HOST: ${{ secrets.API_HOST }} DB_HOST: ${{ secrets.DB_HOST }} DB_PORT: ${{ secrets.DB_PORT }} DB_PASSWORD: ${{ secrets.DB_PASSWORD }} DB: ${{ secrets.DB }} - ES_HOST: ${{ secrets.ES_HOST }} + ES_HOSTS: ${{ secrets.ES_HOSTS }} + ES_USER: ${{ secrets.ES_USER }} + ES_PASSWORD: ${{ secrets.ES_PASSWORD }} ES_PORT: ${{ secrets.ES_PORT }} ENVIRONMENT: ${{ secrets.ENVIRONMENT }} DEBUG: ${{ secrets.DEBUG }} @@ -44,7 +47,7 @@ env: jobs: deploy_to_testing: name: Deploy testing server to google kubernetes engine - if: ${{ github.ref == 'refs/heads/develop'}} + if: ${{ github.ref == 'refs/heads/celery-fixes'}} runs-on: ubuntu-latest environment: name: test diff --git a/deploy/charts/openconceptlab/templates/celery/ocl_celery_beat.yaml b/deploy/charts/openconceptlab/templates/celery/ocl_celery_beat.yaml index c2ffb7a..7d710f8 100644 --- a/deploy/charts/openconceptlab/templates/celery/ocl_celery_beat.yaml +++ b/deploy/charts/openconceptlab/templates/celery/ocl_celery_beat.yaml @@ -63,10 +63,10 @@ spec: value: {{ .Values.app.container.env.dbPassword | quote }} - name: API_HOST - value: {{ .Values.app.container.env.redisHost | quote }} + value: {{ .Values.app.container.env.apiHost | quote }} - name: API_PORT - value: {{ .Values.app.container.env.redisPort | quote }} + value: {{ .Values.service.backendport | quote }} - name: DEBUG value: {{ .Values.app.container.env.debug | quote }} diff --git a/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import.yaml b/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import.yaml index b66816b..ca11f88 100644 --- a/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import.yaml +++ b/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import.yaml @@ -51,4 +51,43 @@ spec: value: {{ .Values.app.container.env.redisPort | quote }} - name: DEBUG - value: {{ .Values.app.container.env.debug | quote }} \ No newline at end of file + value: {{ .Values.app.container.env.debug | quote }} + + - name: DB_HOST + value: {{ .Values.app.container.env.dbHost | quote }} + + - name: DB_PORT + value: {{ .Values.app.container.env.dbPort | quote }} + + - name: DB + value: {{ .Values.app.container.env.db | quote }} + + - name: DB_PASSWORD + value: {{ .Values.app.container.env.dbPassword | quote }} + + - name: API_HOST + value: {{ .Values.app.container.env.apiHost | quote }} + + - name: API_PORT + value: {{ .Values.service.backendport | quote }} + + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.33.2 + command: [ + "/cloud_sql_proxy", + "-instances={{ .Values.app.container.env.databaseInstanceConnectionName }}=tcp:5432", + "-credential_file={{ .Values.app.container.env.googleApplicationCredentialsSecret.filePath }}"] + + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + + volumeMounts: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + mountPath: {{ .Values.app.container.env.googleApplicationCredentialsSecret.mountPath }} + readOnly: true + + volumes: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + secret: + secretName: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} \ No newline at end of file diff --git a/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import_0_1.yaml b/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import_0_1.yaml index 3f05259..952f412 100644 --- a/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import_0_1.yaml +++ b/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import_0_1.yaml @@ -51,4 +51,43 @@ spec: value: {{ .Values.app.container.env.redisPort | quote }} - name: DEBUG - value: {{ .Values.app.container.env.debug | quote }} \ No newline at end of file + value: {{ .Values.app.container.env.debug | quote }} + + - name: DB_HOST + value: {{ .Values.app.container.env.dbHost | quote }} + + - name: DB_PORT + value: {{ .Values.app.container.env.dbPort | quote }} + + - name: DB + value: {{ .Values.app.container.env.db | quote }} + + - name: DB_PASSWORD + value: {{ .Values.app.container.env.dbPassword | quote }} + + - name: API_HOST + value: {{ .Values.app.container.env.apiHost | quote }} + + - name: API_PORT + value: {{ .Values.service.backendport | quote }} + + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.33.2 + command: [ + "/cloud_sql_proxy", + "-instances={{ .Values.app.container.env.databaseInstanceConnectionName }}=tcp:5432", + "-credential_file={{ .Values.app.container.env.googleApplicationCredentialsSecret.filePath }}"] + + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + + volumeMounts: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + mountPath: {{ .Values.app.container.env.googleApplicationCredentialsSecret.mountPath }} + readOnly: true + + volumes: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + secret: + secretName: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} \ No newline at end of file diff --git a/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import_2_3.yaml b/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import_2_3.yaml index 142262e..d744215 100644 --- a/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import_2_3.yaml +++ b/deploy/charts/openconceptlab/templates/celery/ocl_celery_bulk_import_2_3.yaml @@ -51,4 +51,44 @@ spec: value: {{ .Values.app.container.env.redisPort | quote }} - name: DEBUG - value: {{ .Values.app.container.env.debug | quote }} \ No newline at end of file + value: {{ .Values.app.container.env.debug | quote }} + + - name: DB_HOST + value: {{ .Values.app.container.env.dbHost | quote }} + + - name: DB_PORT + value: {{ .Values.app.container.env.dbPort | quote }} + + - name: DB + value: {{ .Values.app.container.env.db | quote }} + + - name: DB_PASSWORD + value: {{ .Values.app.container.env.dbPassword | quote }} + + - name: API_HOST + value: {{ .Values.app.container.env.apiHost | quote }} + + - name: API_PORT + value: {{ .Values.service.backendport | quote }} + + + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.33.2 + command: [ + "/cloud_sql_proxy", + "-instances={{ .Values.app.container.env.databaseInstanceConnectionName }}=tcp:5432", + "-credential_file={{ .Values.app.container.env.googleApplicationCredentialsSecret.filePath }}"] + + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + + volumeMounts: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + mountPath: {{ .Values.app.container.env.googleApplicationCredentialsSecret.mountPath }} + readOnly: true + + volumes: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + secret: + secretName: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} \ No newline at end of file diff --git a/deploy/charts/openconceptlab/templates/celery/ocl_celery_concurrent.yaml b/deploy/charts/openconceptlab/templates/celery/ocl_celery_concurrent.yaml index 7dd3fec..c68366e 100644 --- a/deploy/charts/openconceptlab/templates/celery/ocl_celery_concurrent.yaml +++ b/deploy/charts/openconceptlab/templates/celery/ocl_celery_concurrent.yaml @@ -51,4 +51,43 @@ spec: value: {{ .Values.app.container.env.redisPort | quote }} - name: DEBUG - value: {{ .Values.app.container.env.debug | quote }} \ No newline at end of file + value: {{ .Values.app.container.env.debug | quote }} + + - name: DB_HOST + value: {{ .Values.app.container.env.dbHost | quote }} + + - name: DB_PORT + value: {{ .Values.app.container.env.dbPort | quote }} + + - name: DB + value: {{ .Values.app.container.env.db | quote }} + + - name: DB_PASSWORD + value: {{ .Values.app.container.env.dbPassword | quote }} + + - name: API_HOST + value: {{ .Values.app.container.env.apiHost | quote }} + + - name: API_PORT + value: {{ .Values.service.backendport | quote }} + + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.33.2 + command: [ + "/cloud_sql_proxy", + "-instances={{ .Values.app.container.env.databaseInstanceConnectionName }}=tcp:5432", + "-credential_file={{ .Values.app.container.env.googleApplicationCredentialsSecret.filePath }}"] + + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + + volumeMounts: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + mountPath: {{ .Values.app.container.env.googleApplicationCredentialsSecret.mountPath }} + readOnly: true + + volumes: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + secret: + secretName: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} \ No newline at end of file diff --git a/deploy/charts/openconceptlab/templates/celery/ocl_celery_worker.yaml b/deploy/charts/openconceptlab/templates/celery/ocl_celery_worker.yaml index 320cc08..09a92cb 100644 --- a/deploy/charts/openconceptlab/templates/celery/ocl_celery_worker.yaml +++ b/deploy/charts/openconceptlab/templates/celery/ocl_celery_worker.yaml @@ -51,4 +51,43 @@ spec: value: {{ .Values.app.container.env.redisPort | quote }} - name: DEBUG - value: {{ .Values.app.container.env.debug | quote }} \ No newline at end of file + value: {{ .Values.app.container.env.debug | quote }} + + - name: DB_HOST + value: {{ .Values.app.container.env.dbHost | quote }} + + - name: DB_PORT + value: {{ .Values.app.container.env.dbPort | quote }} + + - name: DB + value: {{ .Values.app.container.env.db | quote }} + + - name: DB_PASSWORD + value: {{ .Values.app.container.env.dbPassword | quote }} + + - name: API_HOST + value: {{ .Values.app.container.env.apiHost | quote }} + + - name: API_PORT + value: {{ .Values.service.backendport | quote }} + + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.33.2 + command: [ + "/cloud_sql_proxy", + "-instances={{ .Values.app.container.env.databaseInstanceConnectionName }}=tcp:5432", + "-credential_file={{ .Values.app.container.env.googleApplicationCredentialsSecret.filePath }}"] + + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + + volumeMounts: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + mountPath: {{ .Values.app.container.env.googleApplicationCredentialsSecret.mountPath }} + readOnly: true + + volumes: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + secret: + secretName: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} \ No newline at end of file diff --git a/deploy/charts/openconceptlab/templates/celery/ocl_celery_worker_indexing.yaml b/deploy/charts/openconceptlab/templates/celery/ocl_celery_worker_indexing.yaml index f210a48..e45a059 100644 --- a/deploy/charts/openconceptlab/templates/celery/ocl_celery_worker_indexing.yaml +++ b/deploy/charts/openconceptlab/templates/celery/ocl_celery_worker_indexing.yaml @@ -51,4 +51,55 @@ spec: value: {{ .Values.app.container.env.redisPort | quote }} - name: DEBUG - value: {{ .Values.app.container.env.debug | quote }} \ No newline at end of file + value: {{ .Values.app.container.env.debug | quote }} + + - name: DB_HOST + value: {{ .Values.app.container.env.dbHost | quote }} + + - name: DB_PORT + value: {{ .Values.app.container.env.dbPort | quote }} + + - name: DB + value: {{ .Values.app.container.env.db | quote }} + + - name: DB_PASSWORD + value: {{ .Values.app.container.env.dbPassword | quote }} + + - name: API_HOST + value: {{ .Values.app.container.env.apiHost | quote }} + + - name: API_PORT + value: {{ .Values.service.backendport | quote }} + + - name: ES_HOSTS + value: {{ .Values.app.container.env.elasticSearchHost | quote }} + + - name: ES_USER + value: {{ .Values.app.container.env.elasticSearchUser | quote }} + + - name: ES_PASSWORD + value: {{ .Values.app.container.env.elasticSearchPassword | quote }} + + - name: ES_SCHEME + value: "https" + + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.33.2 + command: [ + "/cloud_sql_proxy", + "-instances={{ .Values.app.container.env.databaseInstanceConnectionName }}=tcp:5432", + "-credential_file={{ .Values.app.container.env.googleApplicationCredentialsSecret.filePath }}"] + + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + + volumeMounts: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + mountPath: {{ .Values.app.container.env.googleApplicationCredentialsSecret.mountPath }} + readOnly: true + + volumes: + - name: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} + secret: + secretName: {{ .Values.app.container.env.googleApplicationCredentialsSecret.name }} \ No newline at end of file diff --git a/deploy/charts/openconceptlab/templates/oclapi/api_deployment.yaml b/deploy/charts/openconceptlab/templates/oclapi/api_deployment.yaml index 907a640..4621080 100644 --- a/deploy/charts/openconceptlab/templates/oclapi/api_deployment.yaml +++ b/deploy/charts/openconceptlab/templates/oclapi/api_deployment.yaml @@ -63,11 +63,14 @@ spec: - name: DB value: {{ .Values.app.container.env.db | quote }} - - name: ES_HOST + - name: ES_HOSTS value: {{ .Values.app.container.env.elasticSearchHost | quote }} - - name: ES_PORT - value: {{ .Values.app.container.env.elasticSearchPort | quote }} + - name: ES_USER + value: {{ .Values.app.container.env.elasticSearchUser | quote }} + + - name: ES_PASSWORD + value: {{ .Values.app.container.env.elasticSearchPassword | quote }} - name: ES_SCHEME value: "https" diff --git a/deploy/charts/openconceptlab/values.yaml b/deploy/charts/openconceptlab/values.yaml index 894cc95..1efa3c3 100644 --- a/deploy/charts/openconceptlab/values.yaml +++ b/deploy/charts/openconceptlab/values.yaml @@ -19,7 +19,7 @@ image: repository: openconceptlab/oclapi2 pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "production" + tag: "qa" oclweb: port: 4000 @@ -28,7 +28,7 @@ oclweb: repository: openconceptlab/oclweb2 pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "production" + tag: "qa" service: type: NodePort diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 9c52318..32b7981 100644 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -27,12 +27,14 @@ helm upgrade \ --set app.container.env.databaseInstanceConnectionName="${DB_INSTANCE_NAME}"\ --set app.container.env.redisHost="${REDIS_HOST}"\ --set app.container.env.redisPort="${REDIS_PORT}"\ + --set app.container.env.apiHost="${API_HOST}"\ --set app.container.env.dbHost="${DB_HOST}"\ --set app.container.env.dbPort="${DB_PORT}"\ --set app.container.env.db="${DB}"\ --set app.container.env.dbPassword="${DB_PASSWORD}"\ - --set app.container.env.elasticSearchHost="${ES_HOST}"\ - --set app.container.env.elasticSearchPort="${ES_PORT}"\ + --set app.container.env.elasticSearchHost="${ES_HOSTS}"\ + --set app.container.env.elasticSearchUser="${ES_USER}"\ + --set app.container.env.elasticSearchPassword="${ES_PASSWORD}"\ --set app.container.env.flowerPassword="${FLOWER_PASSWORD}"\ --set app.container.env.flowerUser="${FLOWER_USER}"\ --set app.container.env.flowerHost="${FLOWER_HOST}"\