Stream locking (#26)

* Use streaming request in with block to ensure connection is closed.

* Reduce date search range now that IntHub backfill process is implemented.

* Increase number of concurrent connections to new Inthub max minus one.

* Fix expected length in integration_tests.

cdk/downloader_stack.py

@@ -260,7 +260,7 @@ def __init__(
             timeout=core.Duration.minutes(15),
             runtime=aws_lambda.Runtime.PYTHON_3_8,
             environment=downloader_environment_vars,
-            reserved_concurrent_executions=15,
+            reserved_concurrent_executions=24,
         )
 
         aws_logs.LogGroup(

integration_tests/test_link_fetching.py

@@ -43,7 +43,7 @@ def test_that_link_fetching_invocation_executes_correctly(
         assert_that(granules).is_length(68)
 
         granule_counts = db.query(GranuleCount).all()
-        assert_that(granule_counts).is_length(21)
+        assert_that(granule_counts).is_length(5)
 
         statuses = db.query(Status).all()
         assert_that(statuses).is_length(1)
@@ -103,7 +103,7 @@ def test_that_link_fetching_invocation_executes_correctly_when_a_duplicate_granu
         assert_that(granule_we_inserted.download_url).is_equal_to("A download url")
 
         granule_counts = db.query(GranuleCount).all()
-        assert_that(granule_counts).is_length(21)
+        assert_that(granule_counts).is_length(5)
 
         statuses = db.query(Status).all()
         assert_that(statuses).is_length(1)

lambdas/date_generator/handler.py

@@ -8,12 +8,12 @@ def handler(event, context):
 
 def get_dates() -> List[str]:
     """
-    Returns 21 date strings from `datetime.now() - 1 day` with the latest day first
+    Returns 5 date strings from `datetime.now() - 1 day` with the latest day first
     Strings are formatted as %Y-%m-%d
     :returns: List[str] representing 21 days from yesterday
     """
     yesterdays_date = datetime.now().date() - timedelta(days=1)
     return [
         (yesterdays_date - timedelta(days=day)).strftime("%Y-%m-%d")
-        for day in range(0, 21)
+        for day in range(0, 5)
     ]

lambdas/date_generator/tests/test_date_generator_handler.py

@@ -9,7 +9,7 @@
 @freeze_time("2020-01-22")
 def test_that_get_dates_returns_correct_dates():
     expected_dates = [
-        datetime(2020, 1, i).date().strftime("%Y-%m-%d") for i in range(21, 0, -1)
+        datetime(2020, 1, i).date().strftime("%Y-%m-%d") for i in range(21, 16, -1)
     ]
     actual_dates = get_dates()
     assert_that(expected_dates).is_equal_to(actual_dates)
@@ -18,7 +18,7 @@ def test_that_get_dates_returns_correct_dates():
 @freeze_time("2020-04-22")
 def test_that_date_generator_handler_returns_correct_dates():
     expected_dates = [
-        datetime(2020, 4, i).date().strftime("%Y-%m-%d") for i in range(21, 0, -1)
+        datetime(2020, 4, i).date().strftime("%Y-%m-%d") for i in range(21, 16, -1)
     ]
     expected_handler_output = {"query_dates": expected_dates}
     actual_handler_output = handler(None, None)

lambdas/downloader/handler.py

@@ -195,26 +195,26 @@ def download_file(
     with get_session(session_maker) as db:
         try:
             auth = get_scihub_auth(os.environ["USE_INTHUB2"] == "YES")
-            response = requests.get(url=download_url, auth=auth, stream=True)
-            response.raise_for_status()
-
-            aws_checksum = generate_aws_checksum(image_checksum)
-
-            s3_client = get_s3_client()
-            upload_bucket = os.environ["UPLOAD_BUCKET"]
-            root, ext = os.path.splitext(image_filename)
-            zip_key = f"{root}.zip"
-            s3_client.put_object(
-                Body=response.raw.read(),
-                Bucket=upload_bucket,
-                Key=f"{zip_key}",
-                ContentMD5=aws_checksum,
-            )
+            with requests.get(url=download_url, auth=auth, stream=True) as response:
+                response.raise_for_status()
+
+                aws_checksum = generate_aws_checksum(image_checksum)
+
+                s3_client = get_s3_client()
+                upload_bucket = os.environ["UPLOAD_BUCKET"]
+                root, ext = os.path.splitext(image_filename)
+                zip_key = f"{root}.zip"
+                s3_client.put_object(
+                    Body=response.raw.read(),
+                    Bucket=upload_bucket,
+                    Key=f"{zip_key}",
+                    ContentMD5=aws_checksum,
+                )
 
-            granule = db.query(Granule).filter(Granule.id == image_id).first()
-            granule.downloaded = True
-            granule.checksum = image_checksum
-            db.commit()
+                granule = db.query(Granule).filter(Granule.id == image_id).first()
+                granule.downloaded = True
+                granule.checksum = image_checksum
+                db.commit()
         except requests.RequestException as ex:
             error_message = (
                 "Requests exception thrown downloading granule with download_url:"