From 85a376192cfb85af521a533a35d3ae211ef0e8eb Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 20 Nov 2024 18:47:19 +0530 Subject: [PATCH 01/10] Update branch --- openg2p-sr-celery-beat-producers/Dockerfile-git | 2 +- openg2p-sr-celery-workers/Dockerfile-git | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openg2p-sr-celery-beat-producers/Dockerfile-git b/openg2p-sr-celery-beat-producers/Dockerfile-git index 4c19a41..bf7655b 100644 --- a/openg2p-sr-celery-beat-producers/Dockerfile-git +++ b/openg2p-sr-celery-beat-producers/Dockerfile-git @@ -23,7 +23,7 @@ RUN python3 -m venv venv \ RUN python3 -m pip install \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-common \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-auth \ - git+https://github.com/OpenG2P/openg2p-social-registry-bg-tasks@develop\#subdirectory=openg2p-sr-models \ + git+https://github.com/OpenG2P/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ ./src USER ${container_user} diff --git a/openg2p-sr-celery-workers/Dockerfile-git b/openg2p-sr-celery-workers/Dockerfile-git index 7f4ffaf..fc8a5d8 100644 --- a/openg2p-sr-celery-workers/Dockerfile-git +++ b/openg2p-sr-celery-workers/Dockerfile-git @@ -23,7 +23,7 @@ RUN python3 -m venv venv \ RUN python3 -m pip install \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-common \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-auth \ - git+https://github.com/OpenG2P/openg2p-social-registry-bg-tasks@develop\#subdirectory=openg2p-sr-models \ + git+https://github.com/OpenG2P/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ ./src USER ${container_user} From ed55a3840df9cd07488e5250661161f6a65a7c6f Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 20 Nov 2024 19:10:51 +0530 Subject: [PATCH 02/10] Fix column --- openg2p-sr-models/src/openg2p_sr_models/models/id_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openg2p-sr-models/src/openg2p_sr_models/models/id_generation.py b/openg2p-sr-models/src/openg2p_sr_models/models/id_generation.py index e8c910a..017e830 100644 --- a/openg2p-sr-models/src/openg2p_sr_models/models/id_generation.py +++ b/openg2p-sr-models/src/openg2p_sr_models/models/id_generation.py @@ -45,5 +45,5 @@ class G2PQueIDGeneration(BaseORMModelWithTimes): class ResPartner(BaseORMModel): __tablename__ = "res_partner" - id: mapped_column[int] = mapped_column(primary_key=True) + id = mapped_column(Integer, primary_key=True) ref_id = mapped_column(String) From 4281f8c1f1c1159580545367ef2916c7d91b7cdd Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 20 Nov 2024 19:22:24 +0530 Subject: [PATCH 03/10] Add example .env --- openg2p-sr-celery-workers/.env.example | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/openg2p-sr-celery-workers/.env.example b/openg2p-sr-celery-workers/.env.example index 1d5553b..82028f9 100644 --- a/openg2p-sr-celery-workers/.env.example +++ b/openg2p-sr-celery-workers/.env.example @@ -1,4 +1,14 @@ -AUTH_URL=https://keycloak.openg2p.org/realms/master/protocol/openid-connect/token -AUTH_CLIENT_ID=client_id -AUTH_CLIENT_SECRET=client_secret -AUTH_GRANT_TYPE=client_credentials +SR_CELERY_WORKERS_DB_DBNAME: socialregistrydb +SR_CELERY_WORKERS_DB_USERNAME: socialregistryuser +SR_CELERY_WORKERS_DB_HOSTNAME: '172.29.8.235' +SR_CELERY_WORKERS_DB_PORT: 5432 +SR_CELERY_WORKERS_CELERY_BROKER_URL: redis://{{ .Release.Name }}-redis-master:6379/0 +SR_CELERY_WORKERS_CELERY_BACKEND_URL: redis://{{ .Release.Name }}-redis-master:6379/0 +SR_CELERY_WORKERS_AUTH_URL: https://keycloak.openg2p.net/realms/master/protocol/openid-connect/token +SR_CELERY_WORKERS_AUTH_CLIENT_ID: openg2p-sr-loadtest +SR_CELERY_WORKERS_AUTH_CLIENT_SECRET: "" +SR_CELERY_WORKERS_AUTH_GRANT_TYPE: client_credentials +SR_CELERY_WORKERS_MOSIP_GET_UIN_URL: https://idgenerator.loadtest.openg2p.net/v1/idgenerator/uin +SR_CELERY_WORKERS_MOSIP_UPDATE_UIN_URL: https://idgenerator.loadtest.openg2p.net/v1/idgenerator/uin +SR_CELERY_WORKERS_MAX_ID_GENERATION_REQUEST_ATTEMPTS: 3 +SR_CELERY_WORKERS_MAX_ID_GENERATION_UPDATE_ATTEMPTS: 3 \ No newline at end of file From 0fbc3881e6ac2f708e743e0d42d67ed3cd0a69d3 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 20 Nov 2024 19:33:48 +0530 Subject: [PATCH 04/10] Pre-commit --- openg2p-sr-celery-workers/.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openg2p-sr-celery-workers/.env.example b/openg2p-sr-celery-workers/.env.example index 82028f9..aeb9b55 100644 --- a/openg2p-sr-celery-workers/.env.example +++ b/openg2p-sr-celery-workers/.env.example @@ -11,4 +11,4 @@ SR_CELERY_WORKERS_AUTH_GRANT_TYPE: client_credentials SR_CELERY_WORKERS_MOSIP_GET_UIN_URL: https://idgenerator.loadtest.openg2p.net/v1/idgenerator/uin SR_CELERY_WORKERS_MOSIP_UPDATE_UIN_URL: https://idgenerator.loadtest.openg2p.net/v1/idgenerator/uin SR_CELERY_WORKERS_MAX_ID_GENERATION_REQUEST_ATTEMPTS: 3 -SR_CELERY_WORKERS_MAX_ID_GENERATION_UPDATE_ATTEMPTS: 3 \ No newline at end of file +SR_CELERY_WORKERS_MAX_ID_GENERATION_UPDATE_ATTEMPTS: 3 From 81cdb4d5f8c19088207807bcefdefe6bd065e72c Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 20 Nov 2024 20:00:10 +0530 Subject: [PATCH 05/10] temp fix for model change --- openg2p-sr-celery-beat-producers/Dockerfile-git | 2 +- openg2p-sr-celery-workers/Dockerfile-git | 2 +- .../src/openg2p_sr_models/schemas/id_generation.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openg2p-sr-celery-beat-producers/Dockerfile-git b/openg2p-sr-celery-beat-producers/Dockerfile-git index bf7655b..edda067 100644 --- a/openg2p-sr-celery-beat-producers/Dockerfile-git +++ b/openg2p-sr-celery-beat-producers/Dockerfile-git @@ -23,7 +23,7 @@ RUN python3 -m venv venv \ RUN python3 -m pip install \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-common \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-auth \ - git+https://github.com/OpenG2P/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ + git+https://github.com/psnappz/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ ./src USER ${container_user} diff --git a/openg2p-sr-celery-workers/Dockerfile-git b/openg2p-sr-celery-workers/Dockerfile-git index fc8a5d8..ffdf6f4 100644 --- a/openg2p-sr-celery-workers/Dockerfile-git +++ b/openg2p-sr-celery-workers/Dockerfile-git @@ -23,7 +23,7 @@ RUN python3 -m venv venv \ RUN python3 -m pip install \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-common \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-auth \ - git+https://github.com/OpenG2P/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ + git+https://github.com/psnappz/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ ./src USER ${container_user} diff --git a/openg2p-sr-models/src/openg2p_sr_models/schemas/id_generation.py b/openg2p-sr-models/src/openg2p_sr_models/schemas/id_generation.py index d14286e..f1a8706 100644 --- a/openg2p-sr-models/src/openg2p_sr_models/schemas/id_generation.py +++ b/openg2p-sr-models/src/openg2p_sr_models/schemas/id_generation.py @@ -22,5 +22,5 @@ class G2PQueIDGenerationModel(BaseModel): class ResPartnerModel(BaseModel): - registrant_id: str + registrant_id: int ref_id: Optional[str] From 0f1e60df5c1d40a629d592bef2d9b60430e88522 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Thu, 21 Nov 2024 11:09:03 +0530 Subject: [PATCH 06/10] Check response errors for update status API --- openg2p-sr-celery-beat-producers/Dockerfile-git | 2 +- openg2p-sr-celery-workers/Dockerfile-git | 2 +- .../tasks/id_generation_update.py | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/openg2p-sr-celery-beat-producers/Dockerfile-git b/openg2p-sr-celery-beat-producers/Dockerfile-git index edda067..bf7655b 100644 --- a/openg2p-sr-celery-beat-producers/Dockerfile-git +++ b/openg2p-sr-celery-beat-producers/Dockerfile-git @@ -23,7 +23,7 @@ RUN python3 -m venv venv \ RUN python3 -m pip install \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-common \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-auth \ - git+https://github.com/psnappz/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ + git+https://github.com/OpenG2P/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ ./src USER ${container_user} diff --git a/openg2p-sr-celery-workers/Dockerfile-git b/openg2p-sr-celery-workers/Dockerfile-git index ffdf6f4..fc8a5d8 100644 --- a/openg2p-sr-celery-workers/Dockerfile-git +++ b/openg2p-sr-celery-workers/Dockerfile-git @@ -23,7 +23,7 @@ RUN python3 -m venv venv \ RUN python3 -m pip install \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-common \ git+https://github.com/openg2p/openg2p-fastapi-common@1.1.0\#subdirectory=openg2p-fastapi-auth \ - git+https://github.com/psnappz/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ + git+https://github.com/OpenG2P/openg2p-social-registry-bg-tasks@1.0\#subdirectory=openg2p-sr-models \ ./src USER ${container_user} diff --git a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py index 1f127aa..8857c77 100644 --- a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py +++ b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py @@ -79,7 +79,13 @@ def id_generation_update_worker(registrant_id: str): f"MOSIP Update UIN API call failed with status code {response.status_code}" ) - # Update queue entry statuses + # Status code is 200 + if response.json().get("errors"): + raise Exception( + f"MOSIP Update UIN API call failed with error: {response.json().get('errors') or response.json().get('error')}" + ) + + # Status is 200 and No errors then update queue entry statuses queue_entry.number_of_attempts_update += 1 queue_entry.id_generation_update_status = IDGenerationUpdateStatus.COMPLETED queue_entry.last_attempt_datetime = datetime.utcnow() From f474869228d3d9da92ff28b897190d359e4d84d9 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Thu, 21 Nov 2024 11:09:22 +0530 Subject: [PATCH 07/10] Check response errors for update status API --- .../src/openg2p_sr_celery_workers/tasks/id_generation_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py index 8857c77..622f383 100644 --- a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py +++ b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py @@ -82,7 +82,7 @@ def id_generation_update_worker(registrant_id: str): # Status code is 200 if response.json().get("errors"): raise Exception( - f"MOSIP Update UIN API call failed with error: {response.json().get('errors') or response.json().get('error')}" + f"MOSIP Update UIN API call failed with error: {response.json().get('errors')}" ) # Status is 200 and No errors then update queue entry statuses From 4011b437963123fc65a939a7a880d59806178d27 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Thu, 21 Nov 2024 11:11:08 +0530 Subject: [PATCH 08/10] Log the response --- .../openg2p_sr_celery_workers/tasks/id_generation_update.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py index 622f383..c90c565 100644 --- a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py +++ b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py @@ -74,6 +74,9 @@ def id_generation_update_worker(registrant_id: str): response = httpx.put( _config.mosip_update_uin_url, json=update_payload, headers=headers ) + _logger.info( + f"Received response from MOSIP Update UIN API: {response.text}" + ) if response.status_code != 200: raise Exception( f"MOSIP Update UIN API call failed with status code {response.status_code}" @@ -93,7 +96,7 @@ def id_generation_update_worker(registrant_id: str): session.commit() _logger.info( - f"ID generation update completed for registrant_id: {registrant_id}" + f"Mosip update completed for registrant_id: {registrant_id}" ) except Exception as e: From 42db84f49ef0f9ec4d9921926ef326f024237fd2 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Thu, 21 Nov 2024 11:18:46 +0530 Subject: [PATCH 09/10] Fix datetime format issue --- .../tasks/id_generation_update.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py index c90c565..3e31f5b 100644 --- a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py +++ b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py @@ -49,8 +49,6 @@ def id_generation_update_worker(registrant_id: str): f"No UIN found for registrant_id: {registrant_id} in res_partner" ) - uin = res_partner.ref_id - # Get OIDC token access_token = OAuthTokenService.get_component().get_oauth_token() _logger.info("Received access token") @@ -62,13 +60,15 @@ def id_generation_update_worker(registrant_id: str): "Cookie": f"Authorization={access_token}", "Accept": "application/json", } + current_datetime = datetime.utcnow() + formatted_datetime = current_datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' # Call MOSIP Update UIN API to update status update_payload = { "id": "string", "metadata": {}, - "request": {"uin": uin, "status": "ASSIGNED"}, - "requesttime": datetime.utcnow().isoformat(), + "request": {"uin": res_partner.ref_id, "status": "ASSIGNED"}, + "requesttime":formatted_datetime, "version": "string", } response = httpx.put( @@ -95,9 +95,7 @@ def id_generation_update_worker(registrant_id: str): queue_entry.last_attempt_error_code_update = None session.commit() - _logger.info( - f"Mosip update completed for registrant_id: {registrant_id}" - ) + _logger.info(f"Mosip update completed for registrant_id: {registrant_id}") except Exception as e: error_message = f"Error during ID generation update for registrant_id {registrant_id}: {str(e)}" From 03ceb3a52ad0e76cfd4fe7d7e8658de7f3682c55 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Thu, 21 Nov 2024 11:20:44 +0530 Subject: [PATCH 10/10] Pre-commit issues fixed --- .../openg2p_sr_celery_workers/tasks/id_generation_update.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py index 3e31f5b..243a52b 100644 --- a/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py +++ b/openg2p-sr-celery-workers/src/openg2p_sr_celery_workers/tasks/id_generation_update.py @@ -61,14 +61,16 @@ def id_generation_update_worker(registrant_id: str): "Accept": "application/json", } current_datetime = datetime.utcnow() - formatted_datetime = current_datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + formatted_datetime = ( + current_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + ) # Call MOSIP Update UIN API to update status update_payload = { "id": "string", "metadata": {}, "request": {"uin": res_partner.ref_id, "status": "ASSIGNED"}, - "requesttime":formatted_datetime, + "requesttime": formatted_datetime, "version": "string", } response = httpx.put(