catching 404 from PDS

lambdas/handlers/bulk_upload_handler.py

@@ -3,8 +3,7 @@
 from utils.audit_logging_setup import LoggingService
 from utils.decorators.override_error_check import override_error_check
 from utils.decorators.set_audit_arg import set_request_context_for_logging
-from utils.exceptions import (InvalidMessageException,
-                              PdsTooManyRequestsException)
+from utils.exceptions import InvalidMessageException, PdsTooManyRequestsException
 from utils.lloyd_george_validator import LGInvalidFilesException
 
 logger = LoggingService(__name__)

lambdas/models/bulk_upload_status.py

@@ -15,6 +15,7 @@ class UploadStatusBaseClass(BaseModel):
     timestamp: int = Field(default_factory=lambda: int(datetime.now().timestamp()))
     date: str = Field(default_factory=lambda: date_string_yyyymmdd(datetime.now()))
     file_path: str
+    ods_code: str = ""
 
 
 class SuccessfulUpload(UploadStatusBaseClass):
@@ -24,7 +25,6 @@ class SuccessfulUpload(UploadStatusBaseClass):
 class FailedUpload(UploadStatusBaseClass):
     upload_status: Literal["failed"] = "failed"
     failure_reason: str
-    ods_code: str
 
 
 FieldNamesForBulkUploadReport = [
@@ -35,7 +35,7 @@ class FailedUpload(UploadStatusBaseClass):
     "Date",
     "Timestamp",
     "ID",
-    "OdsCode"
+    "OdsCode",
 ]
 
 

lambdas/services/bulk_upload_service.py

@@ -12,19 +12,28 @@
 from services.s3_service import S3Service
 from services.sqs_service import SQSService
 from utils.audit_logging_setup import LoggingService
-from utils.exceptions import (DocumentInfectedException,
-                              InvalidMessageException,
-                              PatientRecordAlreadyExistException,
-                              PdsTooManyRequestsException,
-                              S3FileNotFoundException, TagNotFoundException,
-                              VirusScanFailedException,
-                              VirusScanNoResultException)
-from utils.lloyd_george_validator import (LGInvalidFilesException,
-                                          validate_lg_file_names, validate_with_pds_service,
-                                          getting_patient_info_from_pds)
+from utils.exceptions import (
+    DocumentInfectedException,
+    InvalidMessageException,
+    PatientRecordAlreadyExistException,
+    PdsTooManyRequestsException,
+    S3FileNotFoundException,
+    TagNotFoundException,
+    VirusScanFailedException,
+    VirusScanNoResultException,
+)
+from utils.lloyd_george_validator import (
+    LGInvalidFilesException,
+    validate_lg_file_names,
+    validate_with_pds_service,
+    getting_patient_info_from_pds,
+)
 from utils.request_context import request_context
-from utils.unicode_utils import (contains_accent_char, convert_to_nfc_form,
-                                 convert_to_nfd_form)
+from utils.unicode_utils import (
+    contains_accent_char,
+    convert_to_nfc_form,
+    convert_to_nfd_form,
+)
 from utils.utilities import create_reference_id
 
 logger = LoggingService(__name__)
@@ -69,10 +78,13 @@ def handle_sqs_message(self, message: dict):
             request_context.patient_nhs_no = staging_metadata.nhs_number
             logger.info("Running validation for file names...")
             file_names = [
-                os.path.basename(metadata.file_path) for metadata in staging_metadata.files
+                os.path.basename(metadata.file_path)
+                for metadata in staging_metadata.files
             ]
             validate_lg_file_names(file_names, staging_metadata.nhs_number)
-            pds_patient_details = getting_patient_info_from_pds(staging_metadata.nhs_number)
+            pds_patient_details = getting_patient_info_from_pds(
+                staging_metadata.nhs_number
+            )
             patient_ods_code = pds_patient_details.general_practice_ods
             validate_with_pds_service(file_names, pds_patient_details)
 
@@ -89,10 +101,11 @@ def handle_sqs_message(self, message: dict):
             logger.info("Will stop processing Lloyd George record for this patient.")
 
             failure_reason = str(error)
-            self.report_upload_failure(staging_metadata, failure_reason, patient_ods_code)
+            self.report_upload_failure(
+                staging_metadata, failure_reason, patient_ods_code
+            )
             return
 
-
         logger.info("File validation complete. Checking virus scan results")
 
         try:
@@ -115,7 +128,9 @@ def handle_sqs_message(self, message: dict):
             logger.info("Will stop processing Lloyd George record for this patient")
 
             self.report_upload_failure(
-                staging_metadata, "One or more of the files failed virus scanner check", patient_ods_code
+                staging_metadata,
+                "One or more of the files failed virus scanner check",
+                patient_ods_code,
             )
             return
         except S3FileNotFoundException as e:
@@ -128,7 +143,7 @@ def handle_sqs_message(self, message: dict):
             self.report_upload_failure(
                 staging_metadata,
                 "One or more of the files is not accessible from staging bucket",
-                patient_ods_code
+                patient_ods_code,
             )
             return
 
@@ -155,7 +170,7 @@ def handle_sqs_message(self, message: dict):
             self.report_upload_failure(
                 staging_metadata,
                 "Validation passed but error occurred during file transfer",
-                patient_ods_code
+                patient_ods_code,
             )
             return
 
@@ -207,7 +222,9 @@ def check_virus_result(self, staging_metadata: StagingMetadata):
             f"Verified that all documents for patient {staging_metadata.nhs_number} are clean."
         )
 
-    def put_staging_metadata_back_to_queue(self, staging_metadata: StagingMetadata, patient_ods_code: str):
+    def put_staging_metadata_back_to_queue(
+        self, staging_metadata: StagingMetadata, patient_ods_code: str
+    ):
         if staging_metadata.retries > 14:
             err = "File was not scanned for viruses before maximum retries attempted"
             self.report_upload_failure(staging_metadata, err, patient_ods_code)
@@ -358,13 +375,13 @@ def rollback_transaction(self):
                 f"Failed to rollback the incomplete transaction due to error: {e}"
             )
 
-    def report_upload_complete(self, staging_metadata: StagingMetadata, ods_code: str = ""):
+    def report_upload_complete(
+        self, staging_metadata: StagingMetadata, ods_code: str = ""
+    ):
         nhs_number = staging_metadata.nhs_number
         for file in staging_metadata.files:
             dynamo_record = SuccessfulUpload(
-                nhs_number=nhs_number,
-                file_path=file.file_path,
-                ods_code=ods_code
+                nhs_number=nhs_number, file_path=file.file_path, ods_code=ods_code
             )
             self.dynamo_service.create_item(
                 table_name=self.bulk_upload_report_dynamo_table,
@@ -381,7 +398,7 @@ def report_upload_failure(
                 nhs_number=nhs_number,
                 failure_reason=failure_reason,
                 file_path=file.file_path,
-                ods_code=ods_code
+                ods_code=ods_code,
             )
             self.dynamo_service.create_item(
                 table_name=self.bulk_upload_report_dynamo_table,

lambdas/tests/unit/handlers/test_bulk_upload_handler.py

@@ -1,9 +1,10 @@
 import pytest
 from handlers.bulk_upload_handler import lambda_handler
 from tests.unit.helpers.data.bulk_upload.test_data import (
-    TEST_EVENT_WITH_10_SQS_MESSAGES, TEST_EVENT_WITH_SQS_MESSAGES)
-from utils.exceptions import (InvalidMessageException,
-                              PdsTooManyRequestsException)
+    TEST_EVENT_WITH_10_SQS_MESSAGES,
+    TEST_EVENT_WITH_SQS_MESSAGES,
+)
+from utils.exceptions import InvalidMessageException, PdsTooManyRequestsException
 
 
 @pytest.fixture

lambdas/tests/unit/helpers/data/bulk_upload/expected_bulk_upload_report.csv

@@ -1,3 +1,3 @@
 NhsNumber,UploadStatus,FailureReason,FilePath,Date,Timestamp,ID,OdsCode
-9000000009,complete,,/9000000009/1of1_Lloyd_George_Record_[Joe Bloggs]_[9000000009]_[25-12-2019].pdf,2023-10-30,1698661500,1234-4567-8912-HSDF-TEST,
+9000000009,complete,,/9000000009/1of1_Lloyd_George_Record_[Joe Bloggs]_[9000000009]_[25-12-2019].pdf,2023-10-30,1698661500,1234-4567-8912-HSDF-TEST,Y12345
 9000000025,failed,File name not matching Lloyd George naming convention,/9000000025/invalid_filename.pdf,2023-10-30,1698661500,1234-4567-8912-HSDF-TEST,

lambdas/tests/unit/models/test_bulk_upload_status.py

@@ -9,6 +9,7 @@
     "Date": "2023-10-30",
     "UploadStatus": "complete",
     "FilePath": "/9000000009/1of1_Lloyd_George_Record_[Joe Bloggs]_[9000000009]_[25-12-2019].pdf",
+    "OdsCode": "Y12345",
 }
 
 MOCK_FAILURE_REASON = "File name not matching Lloyd George naming convention"
@@ -33,6 +34,7 @@ def test_create_successful_upload():
         date="2023-10-30",
         upload_status="complete",
         file_path="/9000000009/1of1_Lloyd_George_Record_[Joe Bloggs]_[9000000009]_[25-12-2019].pdf",
+        ods_code="Y12345",
     ).model_dump(by_alias=True)
 
     assert actual == expected
@@ -48,7 +50,7 @@ def test_create_failed_upload():
         upload_status="failed",
         failure_reason=MOCK_FAILURE_REASON,
         file_path="/9000000025/invalid_filename.pdf",
-        ods_code=""
+        ods_code="",
     ).model_dump(by_alias=True)
 
     assert actual == expected
@@ -62,6 +64,7 @@ def test_successful_upload_ids_and_timestamp_are_auto_populated_if_not_given(moc
     actual = SuccessfulUpload(
         nhs_number="9000000009",
         file_path="/9000000009/1of1_Lloyd_George_Record_[Joe Bloggs]_[9000000009]_[25-12-2019].pdf",
+        ods_code="Y12345",
     ).model_dump(by_alias=True)
 
     assert actual == expected
@@ -76,7 +79,7 @@ def test_failed_upload_ids_and_timestamp_are_auto_populated_if_not_given(mocker)
         nhs_number="9000000025",
         file_path="/9000000025/invalid_filename.pdf",
         failure_reason=MOCK_FAILURE_REASON,
-        ods_code=""
+        ods_code="",
     ).model_dump(by_alias=True)
 
     assert actual == expected

lambdas/tests/unit/services/test_bulk_upload_report_service.py

@@ -6,14 +6,22 @@
 from boto3.dynamodb.conditions import Attr
 from freezegun import freeze_time
 from services.bulk_upload_report_service import BulkUploadReportService
-from tests.unit.conftest import (MOCK_BULK_REPORT_TABLE_NAME,
-                                 MOCK_LG_STAGING_STORE_BUCKET)
+from tests.unit.conftest import (
+    MOCK_BULK_REPORT_TABLE_NAME,
+    MOCK_LG_STAGING_STORE_BUCKET,
+)
 from tests.unit.helpers.data.bulk_upload.test_data import readfile
 from tests.unit.helpers.data.dynamo_scan_response import (
-    EXPECTED_RESPONSE, MOCK_EMPTY_RESPONSE, MOCK_RESPONSE,
-    MOCK_RESPONSE_WITH_LAST_KEY, UNEXPECTED_RESPONSE)
+    EXPECTED_RESPONSE,
+    MOCK_EMPTY_RESPONSE,
+    MOCK_RESPONSE,
+    MOCK_RESPONSE_WITH_LAST_KEY,
+    UNEXPECTED_RESPONSE,
+)
 from tests.unit.models.test_bulk_upload_status import (
-    MOCK_DATA_COMPLETE_UPLOAD, MOCK_DATA_FAILED_UPLOAD)
+    MOCK_DATA_COMPLETE_UPLOAD,
+    MOCK_DATA_FAILED_UPLOAD,
+)
 
 
 @pytest.fixture()