From 2a8748058ccd74d9275fc9f0a7c60383cdbb680b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 30 Jan 2023 11:24:08 -0800 Subject: [PATCH 1/3] adding pmanagement config information --- environment_examples/local_env_template.sh | 8 +++ environment_examples/render_local_env.py | 1 + kubetemplates/meshing.yml | 2 +- kubetemplates/pmanagement.yml | 76 +++++++++++++++------- 4 files changed, 64 insertions(+), 23 deletions(-) diff --git a/environment_examples/local_env_template.sh b/environment_examples/local_env_template.sh index 4464c7e..87f9de5 100644 --- a/environment_examples/local_env_template.sh +++ b/environment_examples/local_env_template.sh @@ -231,6 +231,14 @@ export REDIS_PASSWORD=$(kubectl get secret --namespace default redis-release -o export REDIS_HOST="redis-release-master.default.svc.cluster.local" export REDIS_PORT=6379 +#PMANAGEMENT +export PMANAGEMENT_CONFIG_VERSION=1.1 +export PMANAGEMENT_PROOFREADING_TABLE = "proofreading_status_public_v1" +export PMANAGEMENT_PROOFREADINGREVIEW_TABLE = "proofreading_review_public_v1" +export PMANAGEMENT_PROOFREADINGREVIEW_TEST_TABLE = "proofreading_review_public_vtest" +export PMANAGEMENT_NEURONINFORMATION_TABLE = "neuron_information_v2" +export PMANAGEMENT_NEURONINFORMATION_TEST_TABLE = "neuron_information_vtest" +export PMANAGEMENT_DATASTORE_BUCKET_PATH = {{ pmanagement_datastore_bucket_path }} # CELERY export CELERY_WORKER_POOL="celery-pool" diff --git a/environment_examples/render_local_env.py b/environment_examples/render_local_env.py index ef240d0..1bfec2a 100644 --- a/environment_examples/render_local_env.py +++ b/environment_examples/render_local_env.py @@ -33,6 +33,7 @@ def create_spaced_list_of_strings(l): "mat_datastacks": "datastack1,datastack2", "mat_beat_schedule": "{ENV_REPO_PATH}/my_mat_schedule.json", "pcg_graph_ids": "pcg_table1,pcg_table2", + "pmanagement_datastore_bucket_path": "gs://my_bucket", "authservice_secret_key": "randomkey", "global_server": "global.my-dns.com", "guidebook_csrf_key": "random_key", diff --git a/kubetemplates/meshing.yml b/kubetemplates/meshing.yml index 66b46ad..0dc9bd8 100644 --- a/kubetemplates/meshing.yml +++ b/kubetemplates/meshing.yml @@ -24,7 +24,7 @@ spec: name: cpu target: type: Utilization - averageUtilization: 100 + averageUtilization: 80 --- kind: Service apiVersion: v1 diff --git a/kubetemplates/pmanagement.yml b/kubetemplates/pmanagement.yml index 0d1c022..a61b3b0 100644 --- a/kubetemplates/pmanagement.yml +++ b/kubetemplates/pmanagement.yml @@ -1,3 +1,35 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: proofreadingmanagement-config-v${PMANAGEMENT_CONFIG_VERSION} + namespace: default +data: + config.cfg: | + DEBUG = False + TESTING = False + HOME = os.path.expanduser("~") + CORS_ORIGINS = "*" + CORS_HEADERS = "Content-Type" + SECRET_KEY = "b${DASH_SECRET_KEY}" + + LOGGING_FORMAT = '{"source":"%(name)s","time":"%(asctime)s","severity":"%(levelname)s","message":"%(message)s"}' + LOGGING_DATEFORMAT = "%Y-%m-%dT%H:%M:%S.0Z" + LOGGING_LEVEL = logging.DEBUG + JSON_SORT_KEYS = False + datastack_name = "${MAT_DATASTACKS}" + proofreading_table_name = "${PMANAGEMENT_PROOFREADING_TABLE}" + review_table_name = "${PMANAGEMENT_PROOFREADINGREVIEW_TABLE}" + review_test_table_name = "${PMANAGEMENT_PROOFREADINGREVIEW_TEST_TABLE}" + cell_identity_table_name = "${PMANAGEMENT_NEURONINFORMATION_TABLE}" + cell_identity_test_table_name = "${PMANAGEMENT_NEURONINFORMATION_TEST_TABLE}" + datastore_namespace = os.environ.get("DATASTORE_NAMESPACE", "pmanagement") + + datastore_kind = "proofreading_drive" + datastore_project_id = "${PROJECT_NAME}" + pcg_ids = [${PCG_GRAPH_IDS}] + PCG_ADDRESS = "http://pychunkedgraph-service/segmentation/api/v1/table/" + DATASTORE_BUCKET_PATH="${PMANAGEMENT_DATASTORE_BUCKET_PATH}" +--- apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: @@ -137,27 +169,27 @@ spec: jobTemplate: spec: template: - spec: + spec: volumes: - - name: google-cloud-key - secret: - secretName: ${PMANAGEMENT_SERVICE_ACCOUNT_SECRET} + - name: google-cloud-key + secret: + secretName: ${PMANAGEMENT_SERVICE_ACCOUNT_SECRET} containers: - - name: tallyedits - image: ${DOCKER_REPOSITORY}/proofreadingmanagement:v${PMANAGEMENT_VERSION} - imagePullPolicy: Always - args: - - bash - - -c - - python tallyedits/cron_job.py - volumeMounts: - - name: google-cloud-key - mountPath: /home/nginx/.cloudvolume/secrets - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /home/nginx/.cloudvolume/secrets/${GOOGLE_SECRET_FILENAME} - - name: DAF_CREDENTIALS - value: /home/nginx/.cloudvolume/secrets/${CAVE_SECRET_FILENAME} - - name: PCG_GRAPH_IDS - value: ${PCG_GRAPH_IDS} - restartPolicy: OnFailure \ No newline at end of file + - name: tallyedits + image: ${DOCKER_REPOSITORY}/proofreadingmanagement:v${PMANAGEMENT_VERSION} + imagePullPolicy: Always + args: + - bash + - -c + - python tallyedits/cron_job.py + volumeMounts: + - name: google-cloud-key + mountPath: /home/nginx/.cloudvolume/secrets + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /home/nginx/.cloudvolume/secrets/${GOOGLE_SECRET_FILENAME} + - name: DAF_CREDENTIALS + value: /home/nginx/.cloudvolume/secrets/${CAVE_SECRET_FILENAME} + - name: PCG_GRAPH_IDS + value: ${PCG_GRAPH_IDS} + restartPolicy: OnFailure From 8e073a05d3fd68a8f00ead913a68eede12e5d27b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 30 Jan 2023 11:47:24 -0800 Subject: [PATCH 2/3] adding config volume map --- kubetemplates/pmanagement.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kubetemplates/pmanagement.yml b/kubetemplates/pmanagement.yml index a61b3b0..4ba5213 100644 --- a/kubetemplates/pmanagement.yml +++ b/kubetemplates/pmanagement.yml @@ -85,6 +85,9 @@ spec: - name: google-cloud-key secret: secretName: ${PMANAGEMENT_SERVICE_ACCOUNT_SECRET} + - name: proofreadingmanagement-config-volume + configMap: + name: proofreadingmanagement-config-v${PMANAGEMENT_CONFIG_VERSION} terminationGracePeriodSeconds: 10 containers: - name: pmanagement @@ -95,6 +98,8 @@ spec: volumeMounts: - name: google-cloud-key mountPath: /home/nginx/.cloudvolume/secrets + - name: proofreadingmanagement-config-volume + mountPath: /app/proofreadingmanagement/app/instance/ env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /home/nginx/.cloudvolume/secrets/${GOOGLE_SECRET_FILENAME} From 2806c0764d2294d40a8dab0e0e4dea7f74f47b8e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Mar 2023 13:00:10 -0700 Subject: [PATCH 3/3] adding service address to config --- kubetemplates/pmanagement.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubetemplates/pmanagement.yml b/kubetemplates/pmanagement.yml index 4ba5213..5f7e0de 100644 --- a/kubetemplates/pmanagement.yml +++ b/kubetemplates/pmanagement.yml @@ -22,6 +22,7 @@ data: review_test_table_name = "${PMANAGEMENT_PROOFREADINGREVIEW_TEST_TABLE}" cell_identity_table_name = "${PMANAGEMENT_NEURONINFORMATION_TABLE}" cell_identity_test_table_name = "${PMANAGEMENT_NEURONINFORMATION_TEST_TABLE}" + server_address = "https://${GLOBAL_SERVER}" datastore_namespace = os.environ.get("DATASTORE_NAMESPACE", "pmanagement") datastore_kind = "proofreading_drive"