From 5f10f4c0e6ba8166b39d6ef30bc85107e212bcf9 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Wed, 6 Dec 2023 11:56:36 -0500 Subject: [PATCH] Add Lambda to requeue undownloaded granules (#35) Fixes #31 --- .github/workflows/ci.yml | 32 + .github/workflows/deploy-on-release.yml | 19 + Makefile | 5 + cdk/downloader_stack.py | 80 +- lambdas/requeuer/Makefile | 19 + lambdas/requeuer/Pipfile | 25 + lambdas/requeuer/Pipfile.lock | 1025 +++++++++++++++++ lambdas/requeuer/README.md | 107 ++ lambdas/requeuer/handler.py | 93 ++ lambdas/requeuer/invoke | 97 ++ lambdas/requeuer/tests/__init__.py | 0 lambdas/requeuer/tests/conftest.py | 120 ++ lambdas/requeuer/tests/docker-compose.yml | 10 + .../tests/test_requeue_missing_granules.py | 296 +++++ 14 files changed, 1920 insertions(+), 8 deletions(-) create mode 100644 lambdas/requeuer/Makefile create mode 100644 lambdas/requeuer/Pipfile create mode 100644 lambdas/requeuer/Pipfile.lock create mode 100644 lambdas/requeuer/README.md create mode 100644 lambdas/requeuer/handler.py create mode 100755 lambdas/requeuer/invoke create mode 100644 lambdas/requeuer/tests/__init__.py create mode 100644 lambdas/requeuer/tests/conftest.py create mode 100644 lambdas/requeuer/tests/docker-compose.yml create mode 100644 lambdas/requeuer/tests/test_requeue_missing_granules.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a12b4b5..a650b7bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: echo "downloader=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/downloader/Pipfile)" >> $GITHUB_OUTPUT echo "mockscihubproductapi=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_product_api/Pipfile)" >> $GITHUB_OUTPUT echo "mockscihubsearchapi=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_search_api/Pipfile)" >> $GITHUB_OUTPUT + echo "requeuer=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/requeuer/Pipfile)" >> $GITHUB_OUTPUT # echo "db=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/layers/db/Pipfile)" >> $GITHUB_OUTPUT - name: Setup root cache @@ -88,6 +89,13 @@ jobs: path: /home/runner/.local/share/virtualenvs/mock_scihub_search_api-${{ steps.hashes.outputs.mockscihubsearchapi }} key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/mock_scihub_search_api/Pipfile.lock') }} + - name: Setup requeuer cache + uses: actions/cache@v3 + id: requeuer-cache + with: + path: /home/runner/.local/share/virtualenvs/requeuer-${{ steps.hashes.outputs.requeuer }} + key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/requeuer/Pipfile.lock') }} + # - name: Setup db cache # uses: actions/cache@v3 # id: db-cache @@ -130,6 +138,11 @@ jobs: run: | make -C lambdas/mock_scihub_search_api install + - name: Install requeuer dependencies + if: steps.requeuer-cache.outputs.cache-hit != 'true' + run: | + make -C lambdas/requeuer install + - name: Install db dependencies # if: steps.db-cache.outputs.cache-hit != 'true' run: | @@ -168,6 +181,7 @@ jobs: echo "downloader=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/downloader/Pipfile)" >> $GITHUB_OUTPUT echo "mockscihubproductapi=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_product_api/Pipfile)" >> $GITHUB_OUTPUT echo "mockscihubsearchapi=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_search_api/Pipfile)" >> $GITHUB_OUTPUT + echo "requeuer=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/requeuer/Pipfile)" >> $GITHUB_OUTPUT # echo "db=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/layers/db/Pipfile)" >> $GITHUB_OUTPUT - name: Setup root cache @@ -219,6 +233,13 @@ jobs: path: /home/runner/.local/share/virtualenvs/mock_scihub_search_api-${{ steps.hashes.outputs.mockscihubsearchapi }} key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/mock_scihub_search_api/Pipfile.lock') }} + - name: Setup requeuer cache + uses: actions/cache@v3 + id: requeuer-cache + with: + path: /home/runner/.local/share/virtualenvs/requeuer-${{ steps.hashes.outputs.requeuer }} + key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/requeuer/Pipfile.lock') }} + # - name: Setup db cache # uses: actions/cache@v3 # id: db-cache @@ -261,6 +282,11 @@ jobs: run: | make -C lambdas/mock_scihub_search_api install + - name: Install requeuer dependencies + if: steps.requeuer-cache.outputs.cache-hit != 'true' + run: | + make -C lambdas/requeuer install + - name: Install db dependencies # if: steps.db-cache.outputs.cache-hit != 'true' run: | @@ -281,6 +307,12 @@ jobs: PG_DB="test-db" AWS_DEFAULT_REGION="us-east-1" EOF + cat <> lambdas/requeuer/.env + PG_PASSWORD="test-pass" + PG_USER="test-user" + PG_DB="test-db" + AWS_DEFAULT_REGION="us-east-1" + EOF cat <> layers/db/.env PG_PASSWORD="test-pass" PG_USER="test-user" diff --git a/.github/workflows/deploy-on-release.yml b/.github/workflows/deploy-on-release.yml index 5b3af349..e7329038 100644 --- a/.github/workflows/deploy-on-release.yml +++ b/.github/workflows/deploy-on-release.yml @@ -33,6 +33,7 @@ jobs: echo "downloader=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/downloader/Pipfile)" >> $GITHUB_OUTPUT echo "mockscihubproductapi=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_product_api/Pipfile)" >> $GITHUB_OUTPUT echo "mockscihubsearchapi=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_search_api/Pipfile)" >> $GITHUB_OUTPUT + echo "requeuer=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/requeuer/Pipfile)" >> $GITHUB_OUTPUT # echo "db=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/layers/db/Pipfile)" >> $GITHUB_OUTPUT - name: Setup root cache @@ -84,6 +85,13 @@ jobs: path: /home/runner/.local/share/virtualenvs/mock_scihub_search_api-${{ steps.hashes.outputs.mockscihubsearchapi }} key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/mock_scihub_search_api/Pipfile.lock') }} + - name: Setup requeuer cache + uses: actions/cache@v3 + id: requeuer-cache + with: + path: /home/runner/.local/share/virtualenvs/requeuer-${{ steps.hashes.outputs.requeuer }} + key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/requeuer/Pipfile.lock') }} + # - name: Setup db cache # uses: actions/cache@v3 # id: db-cache @@ -126,6 +134,11 @@ jobs: run: | make -C lambdas/mock_scihub_search_api install + - name: Install requeuer dependencies + if: steps.requeuer-cache.outputs.cache-hit != 'true' + run: | + make -C lambdas/requeuer install + - name: Install db dependencies # if: steps.db-cache.outputs.cache-hit != 'true' run: | @@ -144,6 +157,12 @@ jobs: PG_DB="test-db" AWS_DEFAULT_REGION="us-east-1" EOF + cat <> lambdas/requeuer/.env + PG_PASSWORD="test-pass" + PG_USER="test-user" + PG_DB="test-db" + AWS_DEFAULT_REGION="us-east-1" + EOF cat <> layers/db/.env PG_PASSWORD="test-pass" PG_USER="test-user" diff --git a/Makefile b/Makefile index 72a63de5..89dd5945 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ clean: $(MAKE) -C lambdas/downloader clean $(MAKE) -C lambdas/mock_scihub_search_api clean $(MAKE) -C lambdas/mock_scihub_product_api clean + $(MAKE) -C lambdas/requeuer clean $(MAKE) -C alembic_migration clean install: @@ -32,6 +33,7 @@ install: $(MAKE) -C lambdas/downloader install $(MAKE) -C lambdas/mock_scihub_product_api install $(MAKE) -C lambdas/mock_scihub_search_api install + $(MAKE) -C lambdas/requeuer install $(MAKE) -C layers/db install lint: @@ -43,6 +45,7 @@ lint: $(MAKE) -C lambdas/downloader lint $(MAKE) -C lambdas/mock_scihub_search_api lint $(MAKE) -C lambdas/mock_scihub_product_api lint + $(MAKE) -C lambdas/requeuer lint $(MAKE) -C layers/db lint $(MAKE) -C alembic_migration lint @@ -54,6 +57,7 @@ format: $(MAKE) -C lambdas/downloader format $(MAKE) -C lambdas/mock_scihub_search_api format $(MAKE) -C lambdas/mock_scihub_product_api format + $(MAKE) -C lambdas/requeuer format $(MAKE) -C layers/db format $(MAKE) -C alembic_migration format @@ -81,6 +85,7 @@ unit-tests: $(MAKE) -C lambdas/downloader test $(MAKE) -C lambdas/mock_scihub_search_api test $(MAKE) -C lambdas/mock_scihub_product_api test + $(MAKE) -C lambdas/requeuer test $(MAKE) -C layers/db test $(MAKE) -C alembic_migration test diff --git a/cdk/downloader_stack.py b/cdk/downloader_stack.py index 50940253..4411e9fe 100644 --- a/cdk/downloader_stack.py +++ b/cdk/downloader_stack.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Sequence from aws_cdk import ( aws_cloudwatch, @@ -93,11 +93,15 @@ def __init__( if removal_policy_destroy else core.RemovalPolicy.RETAIN, ) + downloader_rds_secret = downloader_rds.secret + + # Make static type checkers happy + assert downloader_rds_secret aws_ssm.StringParameter( self, id=f"{identifier}-downloader-rds-secret-arn", - string_value=downloader_rds.secret.secret_arn, + string_value=downloader_rds_secret.secret_arn, parameter_name=f"/integration_tests/{identifier}/downloader_rds_secret_arn", ) @@ -163,10 +167,10 @@ def __init__( db_layer, psycopg2_layer, ], - environment={"DB_CONNECTION_SECRET_ARN": downloader_rds.secret.secret_arn}, + environment={"DB_CONNECTION_SECRET_ARN": downloader_rds_secret.secret_arn}, ) - downloader_rds.secret.grant_read(migration_function) + downloader_rds_secret.grant_read(migration_function) core.CustomResource( self, @@ -224,7 +228,7 @@ def __init__( link_fetcher_environment_vars = { "STAGE": identifier, "TO_DOWNLOAD_SQS_QUEUE_URL": to_download_queue.queue_url, - "DB_CONNECTION_SECRET_ARN": downloader_rds.secret.secret_arn, + "DB_CONNECTION_SECRET_ARN": downloader_rds_secret.secret_arn, **({"SEARCH_URL": search_url} if search_url else {}), } @@ -280,7 +284,7 @@ def __init__( downloader_environment_vars = { "STAGE": identifier, - "DB_CONNECTION_SECRET_ARN": downloader_rds.secret.secret_arn, + "DB_CONNECTION_SECRET_ARN": downloader_rds_secret.secret_arn, "UPLOAD_BUCKET": upload_bucket, "USE_INTHUB2": "YES" if use_inthub2 else "NO", **({"COPERNICUS_ZIPPER_URL": zipper_url} if zipper_url else {}), @@ -336,8 +340,8 @@ def __init__( self.downloader.role.add_managed_policy(lambda_insights_policy) - downloader_rds.secret.grant_read(link_fetcher) - downloader_rds.secret.grant_read(self.downloader) + downloader_rds_secret.grant_read(link_fetcher) + downloader_rds_secret.grant_read(self.downloader) scihub_credentials = aws_secretsmanager.Secret.from_secret_name_v2( self, @@ -457,3 +461,63 @@ def __init__( id=f"{identifier}-link-fetch-rule", schedule=aws_events.Schedule.expression("cron(0 12 * * ? *)"), ).add_target(aws_events_targets.SfnStateMachine(link_fetcher_step_function)) + + add_requeuer( + self, + identifier=identifier, + secret=downloader_rds_secret, + layers=[db_layer, psycopg2_layer], + queue=to_download_queue, + removal_policy_destroy=removal_policy_destroy, + ) + + +def add_requeuer( + scope: core.Construct, + *, + identifier: str, + layers: Sequence[aws_lambda.ILayerVersion], + removal_policy_destroy: bool, + secret: aws_secretsmanager.ISecret, + queue: aws_sqs.Queue, +) -> None: + # Requeuer Lambda function for manually requeuing undownloaded granules for + # a given date. + requeuer = aws_lambda_python.PythonFunction( + scope, + id=f"{identifier}-requeuer", + entry="lambdas/requeuer", + index="handler.py", + handler="handler", + layers=layers, + memory_size=200, + timeout=core.Duration.minutes(15), + runtime=aws_lambda.Runtime.PYTHON_3_8, + environment={ + "STAGE": identifier, + "TO_DOWNLOAD_SQS_QUEUE_URL": queue.queue_url, + "DB_CONNECTION_SECRET_ARN": secret.secret_arn, + }, + ) + + aws_logs.LogGroup( + scope, + id=f"{identifier}-requeuer-log-group", + log_group_name=f"/aws/lambda/{requeuer.function_name}", + removal_policy=core.RemovalPolicy.DESTROY + if removal_policy_destroy + else core.RemovalPolicy.RETAIN, + retention=aws_logs.RetentionDays.ONE_DAY + if removal_policy_destroy + else aws_logs.RetentionDays.TWO_WEEKS, + ) + + secret.grant_read(requeuer) + queue.grant_send_messages(requeuer) + + core.CfnOutput( + scope, + id=f"{identifier}-requeuer-function-name", + value=requeuer.function_name, + export_name=f"{identifier}-requeuer-function-name", + ) diff --git a/lambdas/requeuer/Makefile b/lambdas/requeuer/Makefile new file mode 100644 index 00000000..ef6bb421 --- /dev/null +++ b/lambdas/requeuer/Makefile @@ -0,0 +1,19 @@ +.PHONEY: clean install lint format test + +clean: + pipenv --rm || true + +install: + pipenv install --dev + +lint: + pipenv run flake8 . + pipenv run isort --check-only --profile black . + pipenv run black --check --diff . + +format: + pipenv run isort --profile black *.py tests/ + pipenv run black *.py tests/ + +test: + pipenv run pytest -v --cov=handler --cov-report term-missing tests/ diff --git a/lambdas/requeuer/Pipfile b/lambdas/requeuer/Pipfile new file mode 100644 index 00000000..ff2266d7 --- /dev/null +++ b/lambdas/requeuer/Pipfile @@ -0,0 +1,25 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +boto3 = "*" +iso8601 = "*" + +[dev-packages] +alembic = "*" +black = "*" +boto3-stubs = {extras = ["lambda", "sqs", "secretsmanager"], version = "*"} +db = {editable = true, path = "./../../layers/db"} +flake8 = "*" +isort = "*" +moto = "*" +mypy = "*" +psycopg2 = "*" +pytest = "*" +pytest-cov = "*" +pytest-docker = "*" + +[requires] +python_version = "3.8" diff --git a/lambdas/requeuer/Pipfile.lock b/lambdas/requeuer/Pipfile.lock new file mode 100644 index 00000000..584352eb --- /dev/null +++ b/lambdas/requeuer/Pipfile.lock @@ -0,0 +1,1025 @@ +{ + "_meta": { + "hash": { + "sha256": "bd6db422bebc22338e1f08801a6800087b01ee6828d453b870b7888eafbd5293" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "boto3": { + "hashes": [ + "sha256:4f62fc1c7f3ea2d22917aa0aa07b86f119abd90bed3d815e4b52fb3d84773e15", + "sha256:b88f0f305186c5fd41f168e006baa45b7002a33029aec8e5bef373237a172fca" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==1.33.6" + }, + "botocore": { + "hashes": [ + "sha256:14282cd432c0683770eee932c43c12bb9ad5730e23755204ad102897c996693a", + "sha256:938056bab831829f90e09ecd70dd6b295afd52b1482f5582ee7a11d8243d9661" + ], + "markers": "python_version >= '3.7'", + "version": "==1.33.6" + }, + "iso8601": { + "hashes": [ + "sha256:6b1d3829ee8921c4301998c909f7829fa9ed3cbdac0d3b16af2d743aed1ba8df", + "sha256:aac4145c4dcb66ad8b648a02830f5e2ff6c24af20f4f482689be402db2429242" + ], + "index": "pypi", + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==2.1.0" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "s3transfer": { + "hashes": [ + "sha256:368ac6876a9e9ed91f6bc86581e319be08188dc60d50e0d56308ed5765446283", + "sha256:c9e56cbe88b28d8e197cf841f1f0c130f246595e77ae5b5a05b69fe7cb83de76" + ], + "markers": "python_version >= '3.7'", + "version": "==0.8.2" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "urllib3": { + "hashes": [ + "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07", + "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0" + ], + "markers": "python_version < '3.10'", + "version": "==1.26.18" + } + }, + "develop": { + "alembic": { + "hashes": [ + "sha256:a23974ea301c3ee52705db809c7413cecd165290c6679b9998dd6c74342ca23a", + "sha256:ab4b3b94d2e1e5f81e34be8a9b7b7575fc9dd5398fccb0bef351ec9b14872623" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==1.13.0" + }, + "attrs": { + "hashes": [ + "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04", + "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015" + ], + "markers": "python_version >= '3.7'", + "version": "==23.1.0" + }, + "black": { + "hashes": [ + "sha256:250d7e60f323fcfc8ea6c800d5eba12f7967400eb6c2d21ae85ad31c204fb1f4", + "sha256:2a9acad1451632021ee0d146c8765782a0c3846e0e0ea46659d7c4f89d9b212b", + "sha256:412f56bab20ac85927f3a959230331de5614aecda1ede14b373083f62ec24e6f", + "sha256:421f3e44aa67138ab1b9bfbc22ee3780b22fa5b291e4db8ab7eee95200726b07", + "sha256:45aa1d4675964946e53ab81aeec7a37613c1cb71647b5394779e6efb79d6d187", + "sha256:4c44b7211a3a0570cc097e81135faa5f261264f4dfaa22bd5ee2875a4e773bd6", + "sha256:4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05", + "sha256:5133f5507007ba08d8b7b263c7aa0f931af5ba88a29beacc4b2dc23fcefe9c06", + "sha256:54caaa703227c6e0c87b76326d0862184729a69b73d3b7305b6288e1d830067e", + "sha256:58e5f4d08a205b11800332920e285bd25e1a75c54953e05502052738fe16b3b5", + "sha256:698c1e0d5c43354ec5d6f4d914d0d553a9ada56c85415700b81dc90125aac244", + "sha256:6c1cac07e64433f646a9a838cdc00c9768b3c362805afc3fce341af0e6a9ae9f", + "sha256:760415ccc20f9e8747084169110ef75d545f3b0932ee21368f63ac0fee86b221", + "sha256:7f622b6822f02bfaf2a5cd31fdb7cd86fcf33dab6ced5185c35f5db98260b055", + "sha256:cf57719e581cfd48c4efe28543fea3d139c6b6f1238b3f0102a9c73992cbb479", + "sha256:d136ef5b418c81660ad847efe0e55c58c8208b77a57a28a503a5f345ccf01394", + "sha256:dbea0bb8575c6b6303cc65017b46351dc5953eea5c0a59d7b7e3a2d2f433a911", + "sha256:fc7f6a44d52747e65a02558e1d807c82df1d66ffa80a601862040a43ec2e3142" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==23.11.0" + }, + "boto3": { + "hashes": [ + "sha256:4f62fc1c7f3ea2d22917aa0aa07b86f119abd90bed3d815e4b52fb3d84773e15", + "sha256:b88f0f305186c5fd41f168e006baa45b7002a33029aec8e5bef373237a172fca" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==1.33.6" + }, + "boto3-stubs": { + "extras": [ + "lambda", + "secretsmanager", + "sqs" + ], + "hashes": [ + "sha256:cd85d6be143e29ec0de74681a236a083f21b5bfb0105f099de109b1cec711806", + "sha256:fba685e81c0aaf59b8827e49fe86eb5836277a472db8478fa34c91ab7fdf5e2c" + ], + "markers": "python_version >= '3.7'", + "version": "==1.33.6" + }, + "botocore": { + "hashes": [ + "sha256:14282cd432c0683770eee932c43c12bb9ad5730e23755204ad102897c996693a", + "sha256:938056bab831829f90e09ecd70dd6b295afd52b1482f5582ee7a11d8243d9661" + ], + "markers": "python_version >= '3.7'", + "version": "==1.33.6" + }, + "botocore-stubs": { + "hashes": [ + "sha256:657f98bb01862d5e60cb15ba04718165d986afb7eda35dad93e488d956669c46", + "sha256:de2e63924addb4a49d8eb3a403b97aaa8ce449987f49f05ce92ea709f42382cb" + ], + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==1.33.6.post1" + }, + "certifi": { + "hashes": [ + "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", + "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474" + ], + "markers": "python_version >= '3.6'", + "version": "==2023.11.17" + }, + "cffi": { + "hashes": [ + "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", + "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", + "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", + "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", + "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", + "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", + "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", + "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", + "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", + "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", + "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", + "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", + "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", + "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", + "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", + "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", + "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", + "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", + "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", + "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", + "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", + "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", + "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", + "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", + "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", + "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", + "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", + "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", + "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", + "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", + "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", + "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", + "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", + "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", + "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", + "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", + "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", + "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", + "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", + "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", + "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", + "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", + "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", + "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", + "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", + "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", + "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", + "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", + "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", + "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", + "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", + "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" + ], + "markers": "python_version >= '3.8'", + "version": "==1.16.0" + }, + "charset-normalizer": { + "hashes": [ + "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", + "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", + "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", + "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", + "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", + "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", + "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", + "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", + "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", + "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", + "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", + "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", + "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", + "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", + "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", + "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", + "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", + "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", + "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", + "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", + "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", + "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", + "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", + "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", + "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", + "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", + "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", + "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", + "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", + "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", + "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", + "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", + "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", + "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", + "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", + "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", + "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", + "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", + "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", + "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", + "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", + "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", + "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", + "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", + "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", + "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", + "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", + "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", + "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", + "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", + "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", + "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", + "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", + "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", + "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", + "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", + "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", + "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", + "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", + "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", + "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", + "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", + "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", + "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", + "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", + "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", + "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", + "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", + "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", + "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", + "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", + "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", + "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", + "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.3.2" + }, + "click": { + "hashes": [ + "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", + "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de" + ], + "markers": "python_version >= '3.7'", + "version": "==8.1.7" + }, + "coverage": { + "extras": [ + "toml" + ], + "hashes": [ + "sha256:0cbf38419fb1a347aaf63481c00f0bdc86889d9fbf3f25109cf96c26b403fda1", + "sha256:12d15ab5833a997716d76f2ac1e4b4d536814fc213c85ca72756c19e5a6b3d63", + "sha256:149de1d2401ae4655c436a3dced6dd153f4c3309f599c3d4bd97ab172eaf02d9", + "sha256:1981f785239e4e39e6444c63a98da3a1db8e971cb9ceb50a945ba6296b43f312", + "sha256:2443cbda35df0d35dcfb9bf8f3c02c57c1d6111169e3c85fc1fcc05e0c9f39a3", + "sha256:289fe43bf45a575e3ab10b26d7b6f2ddb9ee2dba447499f5401cfb5ecb8196bb", + "sha256:2f11cc3c967a09d3695d2a6f03fb3e6236622b93be7a4b5dc09166a861be6d25", + "sha256:307adb8bd3abe389a471e649038a71b4eb13bfd6b7dd9a129fa856f5c695cf92", + "sha256:310b3bb9c91ea66d59c53fa4989f57d2436e08f18fb2f421a1b0b6b8cc7fffda", + "sha256:315a989e861031334d7bee1f9113c8770472db2ac484e5b8c3173428360a9148", + "sha256:3a4006916aa6fee7cd38db3bfc95aa9c54ebb4ffbfc47c677c8bba949ceba0a6", + "sha256:3c7bba973ebee5e56fe9251300c00f1579652587a9f4a5ed8404b15a0471f216", + "sha256:4175e10cc8dda0265653e8714b3174430b07c1dca8957f4966cbd6c2b1b8065a", + "sha256:43668cabd5ca8258f5954f27a3aaf78757e6acf13c17604d89648ecc0cc66640", + "sha256:4cbae1051ab791debecc4a5dcc4a1ff45fc27b91b9aee165c8a27514dd160836", + "sha256:5c913b556a116b8d5f6ef834038ba983834d887d82187c8f73dec21049abd65c", + "sha256:5f7363d3b6a1119ef05015959ca24a9afc0ea8a02c687fe7e2d557705375c01f", + "sha256:630b13e3036e13c7adc480ca42fa7afc2a5d938081d28e20903cf7fd687872e2", + "sha256:72c0cfa5250f483181e677ebc97133ea1ab3eb68645e494775deb6a7f6f83901", + "sha256:7dbc3ed60e8659bc59b6b304b43ff9c3ed858da2839c78b804973f613d3e92ed", + "sha256:88ed2c30a49ea81ea3b7f172e0269c182a44c236eb394718f976239892c0a27a", + "sha256:89a937174104339e3a3ffcf9f446c00e3a806c28b1841c63edb2b369310fd074", + "sha256:9028a3871280110d6e1aa2df1afd5ef003bab5fb1ef421d6dc748ae1c8ef2ebc", + "sha256:99b89d9f76070237975b315b3d5f4d6956ae354a4c92ac2388a5695516e47c84", + "sha256:9f805d62aec8eb92bab5b61c0f07329275b6f41c97d80e847b03eb894f38d083", + "sha256:a889ae02f43aa45032afe364c8ae84ad3c54828c2faa44f3bfcafecb5c96b02f", + "sha256:aa72dbaf2c2068404b9870d93436e6d23addd8bbe9295f49cbca83f6e278179c", + "sha256:ac8c802fa29843a72d32ec56d0ca792ad15a302b28ca6203389afe21f8fa062c", + "sha256:ae97af89f0fbf373400970c0a21eef5aa941ffeed90aee43650b81f7d7f47637", + "sha256:af3d828d2c1cbae52d34bdbb22fcd94d1ce715d95f1a012354a75e5913f1bda2", + "sha256:b4275802d16882cf9c8b3d057a0839acb07ee9379fa2749eca54efbce1535b82", + "sha256:b4767da59464bb593c07afceaddea61b154136300881844768037fd5e859353f", + "sha256:b631c92dfe601adf8f5ebc7fc13ced6bb6e9609b19d9a8cd59fa47c4186ad1ce", + "sha256:be32ad29341b0170e795ca590e1c07e81fc061cb5b10c74ce7203491484404ef", + "sha256:beaa5c1b4777f03fc63dfd2a6bd820f73f036bfb10e925fce067b00a340d0f3f", + "sha256:c0ba320de3fb8c6ec16e0be17ee1d3d69adcda99406c43c0409cb5c41788a611", + "sha256:c9eacf273e885b02a0273bb3a2170f30e2d53a6d53b72dbe02d6701b5296101c", + "sha256:cb536f0dcd14149425996821a168f6e269d7dcd2c273a8bff8201e79f5104e76", + "sha256:d1bc430677773397f64a5c88cb522ea43175ff16f8bfcc89d467d974cb2274f9", + "sha256:d1c88ec1a7ff4ebca0219f5b1ef863451d828cccf889c173e1253aa84b1e07ce", + "sha256:d3d9df4051c4a7d13036524b66ecf7a7537d14c18a384043f30a303b146164e9", + "sha256:d51ac2a26f71da1b57f2dc81d0e108b6ab177e7d30e774db90675467c847bbdf", + "sha256:d872145f3a3231a5f20fd48500274d7df222e291d90baa2026cc5152b7ce86bf", + "sha256:d8f17966e861ff97305e0801134e69db33b143bbfb36436efb9cfff6ec7b2fd9", + "sha256:dbc1b46b92186cc8074fee9d9fbb97a9dd06c6cbbef391c2f59d80eabdf0faa6", + "sha256:e10c39c0452bf6e694511c901426d6b5ac005acc0f78ff265dbe36bf81f808a2", + "sha256:e267e9e2b574a176ddb983399dec325a80dbe161f1a32715c780b5d14b5f583a", + "sha256:f47d39359e2c3779c5331fc740cf4bce6d9d680a7b4b4ead97056a0ae07cb49a", + "sha256:f6e9589bd04d0461a417562649522575d8752904d35c12907d8c9dfeba588faf", + "sha256:f94b734214ea6a36fe16e96a70d941af80ff3bfd716c141300d95ebc85339738", + "sha256:fa28e909776dc69efb6ed975a63691bc8172b64ff357e663a1bb06ff3c9b589a", + "sha256:fe494faa90ce6381770746077243231e0b83ff3f17069d748f645617cefe19d4" + ], + "markers": "python_version >= '3.8'", + "version": "==7.3.2" + }, + "cryptography": { + "hashes": [ + "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960", + "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a", + "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc", + "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a", + "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf", + "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1", + "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39", + "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406", + "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a", + "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a", + "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c", + "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be", + "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15", + "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2", + "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d", + "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157", + "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003", + "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248", + "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a", + "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec", + "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309", + "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7", + "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d" + ], + "markers": "python_version >= '3.7'", + "version": "==41.0.7" + }, + "db": { + "editable": true, + "path": "./../../layers/db" + }, + "exceptiongroup": { + "hashes": [ + "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14", + "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68" + ], + "markers": "python_version < '3.11'", + "version": "==1.2.0" + }, + "flake8": { + "hashes": [ + "sha256:d5b3857f07c030bdb5bf41c7f53799571d75c4491748a3adcd47de929e34cd23", + "sha256:ffdfce58ea94c6580c77888a86506937f9a1a227dfcd15f245d694ae20a6b6e5" + ], + "index": "pypi", + "markers": "python_full_version >= '3.8.1'", + "version": "==6.1.0" + }, + "greenlet": { + "hashes": [ + "sha256:0a02d259510b3630f330c86557331a3b0e0c79dac3d166e449a39363beaae174", + "sha256:0b6f9f8ca7093fd4433472fd99b5650f8a26dcd8ba410e14094c1e44cd3ceddd", + "sha256:100f78a29707ca1525ea47388cec8a049405147719f47ebf3895e7509c6446aa", + "sha256:1757936efea16e3f03db20efd0cd50a1c86b06734f9f7338a90c4ba85ec2ad5a", + "sha256:19075157a10055759066854a973b3d1325d964d498a805bb68a1f9af4aaef8ec", + "sha256:19bbdf1cce0346ef7341705d71e2ecf6f41a35c311137f29b8a2dc2341374565", + "sha256:20107edf7c2c3644c67c12205dc60b1bb11d26b2610b276f97d666110d1b511d", + "sha256:22f79120a24aeeae2b4471c711dcf4f8c736a2bb2fabad2a67ac9a55ea72523c", + "sha256:2847e5d7beedb8d614186962c3d774d40d3374d580d2cbdab7f184580a39d234", + "sha256:28e89e232c7593d33cac35425b58950789962011cc274aa43ef8865f2e11f46d", + "sha256:329c5a2e5a0ee942f2992c5e3ff40be03e75f745f48847f118a3cfece7a28546", + "sha256:337322096d92808f76ad26061a8f5fccb22b0809bea39212cd6c406f6a7060d2", + "sha256:3fcc780ae8edbb1d050d920ab44790201f027d59fdbd21362340a85c79066a74", + "sha256:41bdeeb552d814bcd7fb52172b304898a35818107cc8778b5101423c9017b3de", + "sha256:4eddd98afc726f8aee1948858aed9e6feeb1758889dfd869072d4465973f6bfd", + "sha256:52e93b28db27ae7d208748f45d2db8a7b6a380e0d703f099c949d0f0d80b70e9", + "sha256:55d62807f1c5a1682075c62436702aaba941daa316e9161e4b6ccebbbf38bda3", + "sha256:5805e71e5b570d490938d55552f5a9e10f477c19400c38bf1d5190d760691846", + "sha256:599daf06ea59bfedbec564b1692b0166a0045f32b6f0933b0dd4df59a854caf2", + "sha256:60d5772e8195f4e9ebf74046a9121bbb90090f6550f81d8956a05387ba139353", + "sha256:696d8e7d82398e810f2b3622b24e87906763b6ebfd90e361e88eb85b0e554dc8", + "sha256:6e6061bf1e9565c29002e3c601cf68569c450be7fc3f7336671af7ddb4657166", + "sha256:80ac992f25d10aaebe1ee15df45ca0d7571d0f70b645c08ec68733fb7a020206", + "sha256:816bd9488a94cba78d93e1abb58000e8266fa9cc2aa9ccdd6eb0696acb24005b", + "sha256:85d2b77e7c9382f004b41d9c72c85537fac834fb141b0296942d52bf03fe4a3d", + "sha256:87c8ceb0cf8a5a51b8008b643844b7f4a8264a2c13fcbcd8a8316161725383fe", + "sha256:89ee2e967bd7ff85d84a2de09df10e021c9b38c7d91dead95b406ed6350c6997", + "sha256:8bef097455dea90ffe855286926ae02d8faa335ed8e4067326257cb571fc1445", + "sha256:8d11ebbd679e927593978aa44c10fc2092bc454b7d13fdc958d3e9d508aba7d0", + "sha256:91e6c7db42638dc45cf2e13c73be16bf83179f7859b07cfc139518941320be96", + "sha256:97e7ac860d64e2dcba5c5944cfc8fa9ea185cd84061c623536154d5a89237884", + "sha256:990066bff27c4fcf3b69382b86f4c99b3652bab2a7e685d968cd4d0cfc6f67c6", + "sha256:9fbc5b8f3dfe24784cee8ce0be3da2d8a79e46a276593db6868382d9c50d97b1", + "sha256:ac4a39d1abae48184d420aa8e5e63efd1b75c8444dd95daa3e03f6c6310e9619", + "sha256:b2c02d2ad98116e914d4f3155ffc905fd0c025d901ead3f6ed07385e19122c94", + "sha256:b2d3337dcfaa99698aa2377c81c9ca72fcd89c07e7eb62ece3f23a3fe89b2ce4", + "sha256:b489c36d1327868d207002391f662a1d163bdc8daf10ab2e5f6e41b9b96de3b1", + "sha256:b641161c302efbb860ae6b081f406839a8b7d5573f20a455539823802c655f63", + "sha256:b8ba29306c5de7717b5761b9ea74f9c72b9e2b834e24aa984da99cbfc70157fd", + "sha256:b9934adbd0f6e476f0ecff3c94626529f344f57b38c9a541f87098710b18af0a", + "sha256:ce85c43ae54845272f6f9cd8320d034d7a946e9773c693b27d620edec825e376", + "sha256:cf868e08690cb89360eebc73ba4be7fb461cfbc6168dd88e2fbbe6f31812cd57", + "sha256:d2905ce1df400360463c772b55d8e2518d0e488a87cdea13dd2c71dcb2a1fa16", + "sha256:d57e20ba591727da0c230ab2c3f200ac9d6d333860d85348816e1dca4cc4792e", + "sha256:d6a8c9d4f8692917a3dc7eb25a6fb337bff86909febe2f793ec1928cd97bedfc", + "sha256:d923ff276f1c1f9680d32832f8d6c040fe9306cbfb5d161b0911e9634be9ef0a", + "sha256:daa7197b43c707462f06d2c693ffdbb5991cbb8b80b5b984007de431493a319c", + "sha256:dbd4c177afb8a8d9ba348d925b0b67246147af806f0b104af4d24f144d461cd5", + "sha256:dc4d815b794fd8868c4d67602692c21bf5293a75e4b607bb92a11e821e2b859a", + "sha256:e9d21aaa84557d64209af04ff48e0ad5e28c5cca67ce43444e939579d085da72", + "sha256:ea6b8aa9e08eea388c5f7a276fabb1d4b6b9d6e4ceb12cc477c3d352001768a9", + "sha256:eabe7090db68c981fca689299c2d116400b553f4b713266b130cfc9e2aa9c5a9", + "sha256:f2f6d303f3dee132b322a14cd8765287b8f86cdc10d2cb6a6fae234ea488888e", + "sha256:f33f3258aae89da191c6ebaa3bc517c6c4cbc9b9f689e5d8452f7aedbb913fa8", + "sha256:f7bfb769f7efa0eefcd039dd19d843a4fbfbac52f1878b1da2ed5793ec9b1a65", + "sha256:f89e21afe925fcfa655965ca8ea10f24773a1791400989ff32f467badfe4a064", + "sha256:fa24255ae3c0ab67e613556375a4341af04a084bd58764731972bcbc8baeba36" + ], + "markers": "python_version >= '3'", + "version": "==3.0.1" + }, + "idna": { + "hashes": [ + "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", + "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f" + ], + "markers": "python_version >= '3.5'", + "version": "==3.6" + }, + "importlib-metadata": { + "hashes": [ + "sha256:7fc841f8b8332803464e5dc1c63a2e59121f46ca186c0e2e182e80bf8c1319f7", + "sha256:d97503976bb81f40a193d41ee6570868479c69d5068651eb039c40d850c59d67" + ], + "markers": "python_version < '3.9'", + "version": "==7.0.0" + }, + "importlib-resources": { + "hashes": [ + "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a", + "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6" + ], + "markers": "python_version < '3.9'", + "version": "==6.1.1" + }, + "iniconfig": { + "hashes": [ + "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", + "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.0" + }, + "isort": { + "hashes": [ + "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504", + "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6" + ], + "index": "pypi", + "markers": "python_full_version >= '3.8.0'", + "version": "==5.12.0" + }, + "jinja2": { + "hashes": [ + "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", + "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" + ], + "markers": "python_version >= '3.7'", + "version": "==3.1.2" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "mako": { + "hashes": [ + "sha256:57d4e997349f1a92035aa25c17ace371a4213f2ca42f99bee9a602500cfd54d9", + "sha256:e3a9d388fd00e87043edbe8792f45880ac0114e9c4adc69f6e9bfb2c55e3b11b" + ], + "markers": "python_version >= '3.8'", + "version": "==1.3.0" + }, + "markupsafe": { + "hashes": [ + "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", + "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", + "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", + "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", + "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c", + "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", + "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", + "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb", + "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939", + "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", + "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", + "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", + "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", + "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", + "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", + "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", + "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd", + "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", + "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", + "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", + "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", + "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", + "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", + "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", + "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", + "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007", + "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", + "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", + "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", + "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", + "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", + "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", + "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", + "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1", + "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", + "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", + "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c", + "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", + "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823", + "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", + "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", + "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", + "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", + "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", + "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", + "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", + "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", + "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", + "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", + "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", + "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", + "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", + "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", + "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", + "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", + "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", + "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", + "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc", + "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2", + "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.3" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "moto": { + "hashes": [ + "sha256:2da62d52eaa765dfe2762c920f0a88a58f3a09e04581c91db967d92faec848f1", + "sha256:58c12ab9ee69b6a5d1cddf83611ba4071508f07894317c57844b3ae6dc5bcd38" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==4.2.11" + }, + "mypy": { + "hashes": [ + "sha256:12cce78e329838d70a204293e7b29af9faa3ab14899aec397798a4b41be7f340", + "sha256:1484b8fa2c10adf4474f016e09d7a159602f3239075c7bf9f1627f5acf40ad49", + "sha256:204e0d6de5fd2317394a4eff62065614c4892d5a4d1a7ee55b765d7a3d9e3f82", + "sha256:2643d145af5292ee956aa0a83c2ce1038a3bdb26e033dadeb2f7066fb0c9abce", + "sha256:2c6e4464ed5f01dc44dc9821caf67b60a4e5c3b04278286a85c067010653a0eb", + "sha256:2f7f6985d05a4e3ce8255396df363046c28bea790e40617654e91ed580ca7c51", + "sha256:31902408f4bf54108bbfb2e35369877c01c95adc6192958684473658c322c8a5", + "sha256:40716d1f821b89838589e5b3106ebbc23636ffdef5abc31f7cd0266db936067e", + "sha256:4b901927f16224d0d143b925ce9a4e6b3a758010673eeded9b748f250cf4e8f7", + "sha256:4fc3d14ee80cd22367caaaf6e014494415bf440980a3045bf5045b525680ac33", + "sha256:5cf3f0c5ac72139797953bd50bc6c95ac13075e62dbfcc923571180bebb662e9", + "sha256:6dbdec441c60699288adf051f51a5d512b0d818526d1dcfff5a41f8cd8b4aaf1", + "sha256:72cf32ce7dd3562373f78bd751f73c96cfb441de147cc2448a92c1a308bd0ca6", + "sha256:75aa828610b67462ffe3057d4d8a4112105ed211596b750b53cbfe182f44777a", + "sha256:75c4d2a6effd015786c87774e04331b6da863fc3fc4e8adfc3b40aa55ab516fe", + "sha256:78e25b2fd6cbb55ddfb8058417df193f0129cad5f4ee75d1502248e588d9e0d7", + "sha256:84860e06ba363d9c0eeabd45ac0fde4b903ad7aa4f93cd8b648385a888e23200", + "sha256:8c5091ebd294f7628eb25ea554852a52058ac81472c921150e3a61cdd68f75a7", + "sha256:944bdc21ebd620eafefc090cdf83158393ec2b1391578359776c00de00e8907a", + "sha256:9c7ac372232c928fff0645d85f273a726970c014749b924ce5710d7d89763a28", + "sha256:d9b338c19fa2412f76e17525c1b4f2c687a55b156320acb588df79f2e6fa9fea", + "sha256:ee5d62d28b854eb61889cde4e1dbc10fbaa5560cb39780c3995f6737f7e82120", + "sha256:f2c2521a8e4d6d769e3234350ba7b65ff5d527137cdcde13ff4d99114b0c8e7d", + "sha256:f6efc9bd72258f89a3816e3a98c09d36f079c223aa345c659622f056b760ab42", + "sha256:f7c5d642db47376a0cc130f0de6d055056e010debdaf0707cd2b0fc7e7ef30ea", + "sha256:fcb6d9afb1b6208b4c712af0dafdc650f518836065df0d4fb1d800f5d6773db2", + "sha256:fcd2572dd4519e8a6642b733cd3a8cfc1ef94bafd0c1ceed9c94fe736cb65b6a" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==1.7.1" + }, + "mypy-boto3-lambda": { + "hashes": [ + "sha256:10e0f04168f4064e89ba136218162003f1cb6826dcbfa95ca982d3cb792fc9f7", + "sha256:beac0cb4b94f83a444242db16f601405bdfb6c15808c2c52720224d907e7af40" + ], + "version": "==1.33.0" + }, + "mypy-boto3-secretsmanager": { + "hashes": [ + "sha256:ea765e79988689a2cf6ba9307666aa8a3784f715b371b8fdebcb7694f4e92b9a", + "sha256:f0f1552ed294fd2f09ca38fd1af025149eeadde49500e0fca948ad1ada7d9c3f" + ], + "version": "==1.33.0" + }, + "mypy-boto3-sqs": { + "hashes": [ + "sha256:81f4838e81cbb0c088a10e287922fdf6a3f317cbab6647993ab9dbd567c0e8fb", + "sha256:81f71d5f461e5e670d2ca93df92c93efdd7c29be33eabf8475df5f071e638583" + ], + "version": "==1.33.0" + }, + "mypy-extensions": { + "hashes": [ + "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", + "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782" + ], + "markers": "python_version >= '3.5'", + "version": "==1.0.0" + }, + "packaging": { + "hashes": [ + "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", + "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + ], + "markers": "python_version >= '3.7'", + "version": "==23.2" + }, + "pathspec": { + "hashes": [ + "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20", + "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3" + ], + "markers": "python_version >= '3.7'", + "version": "==0.11.2" + }, + "platformdirs": { + "hashes": [ + "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380", + "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420" + ], + "markers": "python_version >= '3.8'", + "version": "==4.1.0" + }, + "pluggy": { + "hashes": [ + "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12", + "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7" + ], + "markers": "python_version >= '3.8'", + "version": "==1.3.0" + }, + "psycopg2": { + "hashes": [ + "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981", + "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516", + "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3", + "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa", + "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a", + "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693", + "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372", + "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e", + "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59", + "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156", + "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024", + "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913", + "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==2.9.9" + }, + "pycodestyle": { + "hashes": [ + "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f", + "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67" + ], + "markers": "python_version >= '3.8'", + "version": "==2.11.1" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "pyflakes": { + "hashes": [ + "sha256:4132f6d49cb4dae6819e5379898f2b8cce3c5f23994194c24b77d5da2e36f774", + "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc" + ], + "markers": "python_version >= '3.8'", + "version": "==3.1.0" + }, + "pytest": { + "hashes": [ + "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac", + "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==7.4.3" + }, + "pytest-cov": { + "hashes": [ + "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6", + "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==4.1.0" + }, + "pytest-docker": { + "hashes": [ + "sha256:1c17e9202a566f85ed5ef269fe2815bd4899e90eb639622e5d14277372ca7524", + "sha256:7103f97b8c479c826b63d73cfb83383dc1970d35105ed1ce78a722c90c7fe650" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "pyyaml": { + "hashes": [ + "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", + "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", + "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", + "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", + "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", + "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", + "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", + "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", + "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", + "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", + "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", + "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", + "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", + "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", + "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", + "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", + "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", + "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", + "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", + "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", + "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", + "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", + "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", + "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", + "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", + "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", + "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", + "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", + "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", + "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", + "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", + "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", + "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", + "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", + "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", + "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", + "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", + "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", + "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", + "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", + "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", + "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", + "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", + "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", + "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", + "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", + "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", + "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", + "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", + "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" + ], + "markers": "python_version >= '3.6'", + "version": "==6.0.1" + }, + "requests": { + "hashes": [ + "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", + "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" + ], + "markers": "python_version >= '3.7'", + "version": "==2.31.0" + }, + "responses": { + "hashes": [ + "sha256:a2b43f4c08bfb9c9bd242568328c65a34b318741d3fab884ac843c5ceeb543f9", + "sha256:b127c6ca3f8df0eb9cc82fd93109a3007a86acb24871834c47b77765152ecf8c" + ], + "markers": "python_version >= '3.8'", + "version": "==0.24.1" + }, + "s3transfer": { + "hashes": [ + "sha256:368ac6876a9e9ed91f6bc86581e319be08188dc60d50e0d56308ed5765446283", + "sha256:c9e56cbe88b28d8e197cf841f1f0c130f246595e77ae5b5a05b69fe7cb83de76" + ], + "markers": "python_version >= '3.7'", + "version": "==0.8.2" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:02a77eef48da7a5239c003a18afa05c964f1e3001cb2039f69b912b0e0d69c61", + "sha256:0b77e40d63147cd322307a10905f2690661acaa6f21eb1168a6e6de144c97a12", + "sha256:1293cbcaf556f3de5a3eb143012e830a7d78952796f5ba9d2a8286d808e158f1", + "sha256:1373b38a8bba90b54f21ff6b8ec7561d7e4fcc44a1fd70a845ece1014b554f9b", + "sha256:308968eb85969ca3025452cebff7e3d9af5f5c0771b6e19df3c68b1a3c6918ae", + "sha256:364b3d46be78eeaa0efc8771d86bd4e66e0e24bc998610ae9b07ab0630a2e0f2", + "sha256:38a50d4d657bd7aa5a8ddeb06eb4f099c29f9ca7b50295ea0f98793007d448b5", + "sha256:441788cdc1617fe3e43565399c95098d54e91422a049df08acb3709854e7cec0", + "sha256:506ff11bc52426bedb66618d10ec1e41c64667ee685fbffb6a3057e5d9513129", + "sha256:56d33a788c427f29a54600374bb3e435331238e7551c1ce738da5186c20f6c68", + "sha256:5beadd632440aa67f3cb3ec235246c3753f8b3d72b254ee5a87c1e87619952f4", + "sha256:5f326c4264d2f1614f471b6f04e96522f7cc94843172e099bf2fb22079891c20", + "sha256:682961ff4e9fcd9803ab3918c7f8c44ab4076566a0385606377723caf18c371a", + "sha256:6a144c87df1eeeb604e20deb074b9252e7f63b5f528a61b7d9d509c2e67adfb0", + "sha256:6fd3bfc212f68913fe42e9a7b5a39fb259e40e927fe5e813f27c6a692bd624e7", + "sha256:7fdec39fe2495a1c833b917d7c0c8b9d06c0b1b91df74e45be7dc7af325a40fa", + "sha256:8319413aaf11e777ed328a763038c85faf4ff4461a14c09f8c2bf5e46954ea8b", + "sha256:90a8529f04f25051357fc149bc7815b515d018598ff6f1f91038dad665a7ac61", + "sha256:9613ae722a818d231b47fe03c7ff60ce2cd9a54c7a3fb927db9e5df6683c438a", + "sha256:9cfef2ad30c5ee1d494d98f3c55a9ac29ec6d294b70849c541d139e4fe1a74e6", + "sha256:a595fe93ef2722c4877e1db80aabbe172b0af7846c61b2852388780a53203855", + "sha256:a75ac5cdac68c10b71f00aff2f4179168abcf462e73d0289d806293b44abfce6", + "sha256:b8bdfb73d07467f2e21e7ff3abc823d52f88b1e5c377fc14da625b30469350ab", + "sha256:bc626e44fec23d9ea92aeecd2359720e8620c1f963c8e24bfdd27e757ed0548c", + "sha256:bd73da5de31118a8130540297779d36bf4d7414c6cca8d7f769b1550dafce78d", + "sha256:befa0b60b663fdbc1bb1bde60d3788ff5a64700f253f7981a22081f3b44239f2", + "sha256:c101e9f57d8a67a4b613852d4a5ee850cd2e8b4791ddba2a90ced4dbc66e5fa2", + "sha256:c272f0340a40d178461b2b54f27360289e063f70db495daa852c2f318fc00640", + "sha256:ce33a952476f9100daa76fb8228fdc99ac11df3c316be2eb946ba31fbe845ba6", + "sha256:dd940003b5724e7376dd627b13086798076c5bc124d562163224334854bdd0ca", + "sha256:ddb17736fc2999dc4e550f02e05add7a2197668cde059269b23989d8730ef71a", + "sha256:e04efa8dd75b9bfc16a6bc174e715678c6e99f52c633eccef76e156e408a5432", + "sha256:e7902051dc747cc96b552230464ddb2c96407e7f07680c71c1923dca2f3a6d9d", + "sha256:faad6bcbc1af9dfb2b2e02be988f992989d99e3eae0c5b21fce818d47aab5181" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.4.0" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + }, + "types-awscrt": { + "hashes": [ + "sha256:850d5ad95d8f337b15fb154790f39af077faf5c08d43758fd750f379a87d5f73", + "sha256:a577c4d60a7fb7e21b436a73207a66f6ba50329d578b347934c5d99d4d612901" + ], + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==0.19.19" + }, + "types-s3transfer": { + "hashes": [ + "sha256:2e41756fcf94775a9949afa856489ac4570308609b0493dfbd7b4d333eb423e6", + "sha256:5e084ebcf2704281c71b19d5da6e1544b50859367d034b50080d5316a76a9418" + ], + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==0.8.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0", + "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef" + ], + "markers": "python_version < '3.11'", + "version": "==4.8.0" + }, + "urllib3": { + "hashes": [ + "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07", + "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0" + ], + "markers": "python_version < '3.10'", + "version": "==1.26.18" + }, + "werkzeug": { + "hashes": [ + "sha256:507e811ecea72b18a404947aded4b3390e1db8f826b494d76550ef45bb3b1dcc", + "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10" + ], + "markers": "python_version >= '3.8'", + "version": "==3.0.1" + }, + "xmltodict": { + "hashes": [ + "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56", + "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" + ], + "markers": "python_version >= '3.4'", + "version": "==0.13.0" + }, + "zipp": { + "hashes": [ + "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31", + "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0" + ], + "markers": "python_version >= '3.8'", + "version": "==3.17.0" + } + } +} diff --git a/lambdas/requeuer/README.md b/lambdas/requeuer/README.md new file mode 100644 index 00000000..965da14a --- /dev/null +++ b/lambdas/requeuer/README.md @@ -0,0 +1,107 @@ +# Requeuer 🔗 + +The Requeuer is a standalone AWS Lambda function that will requeue granules with +a specified ingestion date, if they were not downloaded. This allows us to +backfill missing granules where messages went to the DLQ, but expired before the +DLQ was redriven. + +## Requeueing Missing Granules + +Although this Lambda function is deployed as part of this repository's CDK +stack, it is not connected to anything, and is thus "standalone," requiring +manual execution. Therefore, once deployed, you may trigger execution manually +either via the AWS Console or the AWS CLI. + +The input is simply a single date to use for selecting granules in the database +with a matching value for `ingestiondate` (and with `downloaded` set to `False`), +along with a "dry run" indicator, and must be in the following form: + +```json +{ + "dry_run": true | false, + "date": "YYYY-MM-DD" +} +``` + +When `"dry_run"` is set to `true`, the missing (undownloaded) granules for the +specified date will be found, but they will _not_ be requeued for download. + +Note that a value for `"dry_run"` is _required_ as a precaution to reduce the +likelihood of accidental requeuing, as might be the case if it were to default +to `false`, when not supplied. Conversely, this may also reduce the chance of +confusion as to why granules are _not_ requeued, if it were to instead default +to `true`. + +To make it easy to invoke the Lambda function from the command line, you may use +the provided `invoke` shell script. For example, to perform a "dry run" for +granules ingested on June 10, 2023, you may run the script as follows: + +```plain +./invoke --dry-run 2023-06-10 response.json +``` + +The output will be written to the indicated file (`response.json` in this case), +and will include the given inputs, as well as a list of all granules ingested +on the specified date that have not been downloaded. + +**NOTE:** It is likely more convenient to run `invoke` from the root of the +repository, where you likely have a `.env` file with the `IDENTIFIER` +environment variable defined. The `invoke` script will automatically source a +`.env` found in the current directory. Therefore, from the root of the +repository, it might be more convenient to run the Lambda function as follows: + +```plain +lambdas/requeuer/invoke --dry-run 2023-06-10 response.json +``` + +## Development + +This Lambda makes use of `pipenv` for managing depedencies and for building the +function when deploying it. + +To get setup for developing this project, run: + +```bash +make install +``` + +This Lambda makes use of the `db` module that will be available via a Lambda +Layer once deployed. For local development purposes, it is installed as an +editable relative `[dev-packages]` dependency + +### .env + +This Lambda requires a `.env` file in its directory containing the following env +vars: + +```plain +PG_PASSWORD="" +PG_USER="" +PG_DB="" +``` + +This is used whilst running the tests to provide both the Postgres container and +the test code the credentials needed to access the database created. + +### Makefile + +A `Makefile` is provided to abstract away commonly used commands: + +**`make install`** + +> This will run `pipenv install --dev` to install development dependencies + +**`make lint`** + +> This will perform a dry run of `flake8`, `isort`, and `black` and let you know +> what issues were found + +**`make format`** + +> This will peform a run of `isort` and `black`, this **will** modify files if +> issues were found + +**`make test`** + +> This will run the unit tests of the project with `pytest` using the contents +> of your `.env` file diff --git a/lambdas/requeuer/handler.py b/lambdas/requeuer/handler.py new file mode 100644 index 00000000..2f9af7c1 --- /dev/null +++ b/lambdas/requeuer/handler.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import datetime +import json +import os +from re import T +from typing import TYPE_CHECKING, Any, Callable, Mapping, Sequence, TypedDict + +import iso8601 +from db.models.granule import Granule +from sqlalchemy import func # type: ignore +from sqlalchemy.orm import Session # type: ignore +from typing_extensions import TypeAlias + +if TYPE_CHECKING: + from mypy_boto3_sqs import SQSClient + +SessionMaker: TypeAlias = Callable[[], Session] + + +class GranuleMessage(TypedDict): + id: str + filename: str + download_url: str + + +class Response(TypedDict): + dry_run: bool + queue_url: str + ingestion_date: str + count: int + granules: Sequence[GranuleMessage] + + +def handler(event: Mapping[str, Any], context: Any) -> Response: + import boto3 + from db.session import get_session_maker + + print(json.dumps(event)) + + queue_url = os.environ["TO_DOWNLOAD_SQS_QUEUE_URL"] + response = _handler(event, get_session_maker(), boto3.client("sqs"), queue_url) + + print(json.dumps(response)) + + return response + + +def _handler( + event: Mapping[str, Any], + make_session: SessionMaker, + sqs_client: SQSClient, + queue_url: str, +) -> Response: + if (dry_run := event["dry_run"]) not in [True, False]: + raise TypeError("dry_run must be a boolean") + + dt = iso8601.parse_date(event["date"]) + date = datetime.date(dt.year, dt.month, dt.day) + messages = tuple(map(granule_message, select_missing_granules(date, make_session))) + + if not dry_run: + for message in messages: + sqs_client.send_message(QueueUrl=queue_url, MessageBody=json.dumps(message)) + + return { + "dry_run": dry_run, + "queue_url": queue_url, + "ingestion_date": f"{date}", + "count": len(messages), + "granules": messages, + } + + +def select_missing_granules( + ingestion_date: datetime.date, + Session: SessionMaker, +) -> Sequence[Granule]: + conditions = ( + Granule.downloaded == False, # noqa: E712 + func.date_trunc("day", Granule.ingestiondate) == ingestion_date, + ) + + with Session() as session: + return session.query(Granule).filter(*conditions).all() # type: ignore + + +def granule_message(granule: Granule) -> GranuleMessage: + return GranuleMessage( + id=granule.id, + filename=granule.filename, + download_url=granule.download_url, + ) diff --git a/lambdas/requeuer/invoke b/lambdas/requeuer/invoke new file mode 100755 index 00000000..03ca2d04 --- /dev/null +++ b/lambdas/requeuer/invoke @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# Source the .env file, if it exists, to get the IDENTIFIER environment variable. +test -f "${PWD}/.env" && builtin source "$_" + +usage() { + echo "Usage: $0 --[no]-dry-run YYYY-MM-DD OUTFILE" 1>&2 + echo "" 1>&2 + echo " --[no]-dry-run Whether or not to run in dry run mode (i.e., list, but don't requeue, missing granules)." 1>&2 + echo " YYYY-MM-DD Ingestion date of the undownloaded granules to requeue." 1>&2 + echo " OUTFILE Where to write requeuer output (JSON)." 1>&2 + echo "" 1>&2 + exit 1 +} + +if [[ -z "${IDENTIFIER}" ]]; then + echo "ERROR: The IDENTIFIER environment variable must be set." 1>&2 + echo "" 1>&2 + usage +fi + +n_args=3 + +if [[ $# -ne $n_args ]]; then + if [[ $# -gt 0 ]]; then + echo "ERROR: You must specify exactly ${n_args} arguments." 1>&2 + echo "" 1>&2 + fi + + usage +fi + +if [[ "$1" != "--dry-run" && "$1" != "--no-dry-run" ]]; then + echo "ERROR: The first argument must be either --dry-run or --no-dry-run" 1>&2 + echo "" 1>&2 + usage +fi + +if [[ "$1" == "--dry-run" ]]; then + dry_run="true" +else + dry_run="false" +fi + +if [[ ! "$2" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then + echo "ERROR: Invalid ingestion date format (YYYY-MM-DD): $2" 1>&2 + echo "" 1>&2 + usage +fi + +ingestion_date=$2 +outfile=$3 + +function_name=$( + aws cloudformation describe-stacks \ + --stack-name "hls-s2-downloader-serverless-${IDENTIFIER}" \ + --query "Stacks[0].Outputs[?ExportName=='${IDENTIFIER}-requeuer-function-name'].OutputValue" \ + --output text +) + +if [[ ${dry_run} == "true" ]]; then + echo "----- DRY RUN -----" 1>&2 +fi + +echo "Invoking Lambda function ${function_name}" 1>&2 + +response=$( + aws lambda invoke \ + --function-name "${function_name}" \ + --payload '{"dry_run": '"${dry_run}"', "date": "'"${ingestion_date}"'"}' \ + --cli-binary-format raw-in-base64-out \ + "${outfile}" +) + +if [[ ${response} =~ "FunctionError" ]]; then + echo "ERROR: Lambda function failed: $(<"${outfile}")." 1>&2 + echo "" 1>&2 + exit 1 +elif [[ ${dry_run} == "false" ]]; then + echo "Requeued undownloaded granules with an ingestion date of ${ingestion_date}." 1>&2 + echo "The list of granules was written to ${outfile}." 1>&2 + echo "To see the logs for the requeuer, run the following command:" 1>&2 + echo "" 1>&2 + echo " aws logs tail --follow /aws/lambda/${function_name}" 1>&2 + echo "" 1>&2 +else + echo "Would requeue undownloaded granules with an ingestion date of ${ingestion_date}." 1>&2 + echo "The list of granules that would be requeued was written to ${outfile}." 1>&2 + echo "To requeue them, run the following command:" 1>&2 + echo "" 1>&2 + echo " $0 --no-dry-run $ingestion_date ${outfile}" 1>&2 + echo "" 1>&2 + echo "Note that this will rerun the query for ${ingestion_date}, ignoring and" 1>&2 + echo "overwriting the current contents of ${outfile}." 1>&2 +fi diff --git a/lambdas/requeuer/tests/__init__.py b/lambdas/requeuer/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lambdas/requeuer/tests/conftest.py b/lambdas/requeuer/tests/conftest.py new file mode 100644 index 00000000..404b5063 --- /dev/null +++ b/lambdas/requeuer/tests/conftest.py @@ -0,0 +1,120 @@ +import json +import os +import pathlib +from contextlib import contextmanager +from typing import cast + +import alembic.command +import alembic.config +import boto3 +import pytest +from _pytest.monkeypatch import MonkeyPatch +from moto import mock_secretsmanager, mock_sqs +from mypy_boto3_secretsmanager.client import SecretsManagerClient +from mypy_boto3_sqs.service_resource import SQSServiceResource +from sqlalchemy import create_engine # type: ignore +from sqlalchemy.engine import Engine, Transaction, url # type: ignore +from sqlalchemy.exc import OperationalError # type: ignore +from sqlalchemy.orm import Session # type: ignore + +UNIT_TEST_DIR = pathlib.Path(__file__).parent + + +def check_pg_status(engine: Engine) -> bool: + try: + engine.execute("SELECT * FROM pg_catalog.pg_tables;") + return True + except OperationalError: + return False + + +@pytest.fixture(scope="session") +def postgres_engine(docker_ip, docker_services, db_connection_secret: None): + db_url = url.URL.create( + "postgresql", + username=os.environ["PG_USER"], + password=os.environ["PG_PASSWORD"], + host="localhost", + database=os.environ["PG_DB"], + ) + pg_engine = cast(Engine, create_engine(db_url)) + docker_services.wait_until_responsive( + timeout=15.0, pause=1, check=lambda: check_pg_status(pg_engine) + ) + + alembic_root = UNIT_TEST_DIR.parent.parent.parent / "alembic_migration" + alembic_config = alembic.config.Config(str(alembic_root / "alembic.ini")) + alembic_config.set_main_option("script_location", str(alembic_root)) + alembic.command.upgrade(alembic_config, "head") + + return pg_engine + + +@pytest.fixture +def db_session(postgres_engine: Engine): + with postgres_engine.connect() as connection: + with cast(Transaction, connection.begin()) as transaction: + with Session(bind=connection) as session: + yield session + transaction.rollback() + + +@pytest.fixture(autouse=True) +def aws_credentials(monkeysession: MonkeyPatch): + monkeysession.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeysession.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeysession.setenv("AWS_SECURITY_TOKEN", "testing") + monkeysession.setenv("AWS_SESSION_TOKEN", "testing") + monkeysession.setenv("AWS_DEFAULT_REGION", "us-east-1") + + +@pytest.fixture +def sqs_resource(): + with mock_sqs(): + yield boto3.resource("sqs") + + +@pytest.fixture +def sqs_client(): + with mock_sqs(): + yield boto3.client("sqs") + + +@pytest.fixture +def sqs_queue(sqs_resource: SQSServiceResource): + return sqs_resource.create_queue(QueueName="mock-queue") + + +@pytest.fixture(scope="session") +def secrets_manager_client(): + with mock_secretsmanager(): + yield boto3.client("secretsmanager") + + +@pytest.fixture(scope="session") +def db_connection_secret( + secrets_manager_client: SecretsManagerClient, monkeysession: MonkeyPatch +): + arn = secrets_manager_client.create_secret( + Name="db-connection", + SecretString=json.dumps( + { + "username": os.environ["PG_USER"], + "password": os.environ["PG_PASSWORD"], + "host": "localhost", + "dbname": os.environ["PG_DB"], + } + ), + )["ARN"] + monkeysession.setenv("DB_CONNECTION_SECRET_ARN", arn) + + +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig: pytest.Config): + return UNIT_TEST_DIR / "docker-compose.yml" + + +@pytest.fixture(scope="session") +def monkeysession(request: pytest.FixtureRequest): + with MonkeyPatch().context() as mp: + yield mp diff --git a/lambdas/requeuer/tests/docker-compose.yml b/lambdas/requeuer/tests/docker-compose.yml new file mode 100644 index 00000000..ff7b2f16 --- /dev/null +++ b/lambdas/requeuer/tests/docker-compose.yml @@ -0,0 +1,10 @@ +version: "3.9" +services: + postgres: + image: postgres:10.12-alpine + environment: + POSTGRES_PASSWORD: "${PG_PASSWORD}" + POSTGRES_USER: "${PG_USER}" + POSTGRES_DB: "${PG_DB}" + ports: + - 5432:5432 diff --git a/lambdas/requeuer/tests/test_requeue_missing_granules.py b/lambdas/requeuer/tests/test_requeue_missing_granules.py new file mode 100644 index 00000000..bb77639f --- /dev/null +++ b/lambdas/requeuer/tests/test_requeue_missing_granules.py @@ -0,0 +1,296 @@ +import json +from datetime import date, datetime + +import pytest +from db.models.granule import Granule +from handler import Response, _handler +from mypy_boto3_sqs.client import SQSClient +from mypy_boto3_sqs.service_resource import Queue +from sqlalchemy.orm import Session # type: ignore + + +def test_missing_dry_run_raises( + db_session: Session, + sqs_client: SQSClient, + sqs_queue: Queue, +): + with pytest.raises(KeyError, match="dry_run"): + _handler( + dict(date="2021-01-01"), + lambda: db_session, + sqs_client, + sqs_queue.url, + ) + + +def test_missing_date_raises( + db_session: Session, + sqs_client: SQSClient, + sqs_queue: Queue, +): + with pytest.raises(KeyError, match="date"): + _handler( + dict(dry_run=False), + lambda: db_session, + sqs_client, + sqs_queue.url, + ) + + +def test_invalid_dry_run_raises( + db_session: Session, + sqs_client: SQSClient, + sqs_queue: Queue, +): + with pytest.raises(TypeError, match="dry_run"): + _handler( + dict(dry_run="foo", date="2021-01-01"), + lambda: db_session, + sqs_client, + sqs_queue.url, + ) + + +def test_invalid_date_raises( + db_session: Session, + sqs_client: SQSClient, + sqs_queue: Queue, +): + with pytest.raises(ValueError, match="date"): + _handler( + dict(dry_run=False, date="foo"), + lambda: db_session, + sqs_client, + sqs_queue.url, + ) + + +def test_dry_run_doesnt_enqueue( + db_session: Session, + sqs_client: SQSClient, + sqs_queue: Queue, +): + ingestion_mdy = (2021, 1, 1) + ingestion_datetime = datetime(*ingestion_mdy, 12, 0, 0) + ingestion_date = date(*ingestion_mdy) + + db_session.add( + Granule( + id="foo", + filename="foo.tif", + tileid="foo", + size=100, + beginposition=ingestion_datetime, + endposition=ingestion_datetime, + ingestiondate=ingestion_datetime, + download_url="https://example.com/foo.tif", + downloaded=False, + ), # type: ignore + ) + db_session.add( + Granule( + id="bar", + filename="bar.tif", + tileid="bar", + size=100, + beginposition=ingestion_datetime, + endposition=ingestion_datetime, + ingestiondate=ingestion_datetime, + download_url="https://example.com/bar.tif", + downloaded=False, + ), # type: ignore + ) + db_session.commit() + + actual = _handler( + dict(dry_run=True, date=f"{ingestion_date}"), + lambda: db_session, + sqs_client, + sqs_queue.url, + ) + expected = dict( + dry_run=True, + queue_url=sqs_queue.url, + ingestion_date="2021-01-01", + count=2, + granules=( + { + "id": "foo", + "filename": "foo.tif", + "download_url": "https://example.com/foo.tif", + }, + { + "id": "bar", + "filename": "bar.tif", + "download_url": "https://example.com/bar.tif", + }, + ), + ) + received = sqs_client.receive_message( + QueueUrl=sqs_queue.url, MaxNumberOfMessages=10 + ) + + assert actual == expected + assert "Messages" not in received + + +def test_none_missing_for_date( + db_session: Session, + sqs_client: SQSClient, + sqs_queue: Queue, +): + actual = _handler( + dict(dry_run=False, date="2021-01-01"), + lambda: db_session, + sqs_client, + sqs_queue.url, + ) + expected = dict( + dry_run=False, + queue_url=sqs_queue.url, + ingestion_date="2021-01-01", + count=0, + granules=(), + ) + received = sqs_client.receive_message( + QueueUrl=sqs_queue.url, MaxNumberOfMessages=10 + ) + + assert actual == expected + assert "Messages" not in received + + +def test_some_missing_for_date( + db_session: Session, + sqs_client: SQSClient, + sqs_queue: Queue, +): + ingestion_mdy = (2021, 1, 1) + ingestion_datetime = datetime(*ingestion_mdy, 12, 0, 0) + ingestion_date = date(*ingestion_mdy) + + db_session.add( + Granule( + id="foo", + filename="foo.tif", + tileid="foo", + size=100, + beginposition=ingestion_datetime, + endposition=ingestion_datetime, + ingestiondate=ingestion_datetime, + download_url="https://example.com/foo.tif", + downloaded=False, + ), # type: ignore + ) + db_session.add( + Granule( + id="bar", + filename="bar.tif", + tileid="bar", + size=100, + beginposition=ingestion_datetime, + endposition=ingestion_datetime, + ingestiondate=ingestion_datetime, + download_url="https://example.com/bar.tif", + downloaded=True, + ), # type: ignore + ) + db_session.commit() + + actual = _handler( + dict(dry_run=False, date=f"{ingestion_date}"), + lambda: db_session, + sqs_client, + sqs_queue.url, + ) + expected = Response( + dry_run=False, + queue_url=sqs_queue.url, + ingestion_date=f"{ingestion_date}", + count=1, + granules=( + { + "id": "foo", + "filename": "foo.tif", + "download_url": "https://example.com/foo.tif", + }, + ), + ) + received = sqs_client.receive_message( + QueueUrl=sqs_queue.url, MaxNumberOfMessages=10 + ) + + assert actual == expected + assert len(received["Messages"]) == 1 + assert received["Messages"][0].get("Body") == json.dumps(expected["granules"][0]) + + +def test_all_missing_for_date( + db_session: Session, + sqs_client: SQSClient, + sqs_queue: Queue, +): + ingestion_mdy = (2021, 1, 1) + ingestion_datetime = datetime(*ingestion_mdy, 12, 0, 0) + ingestion_date = date(*ingestion_mdy) + + db_session.add( + Granule( + id="foo", + filename="foo.tif", + tileid="foo", + size=100, + beginposition=ingestion_datetime, + endposition=ingestion_datetime, + ingestiondate=ingestion_datetime, + download_url="https://example.com/foo.tif", + downloaded=False, + ), # type: ignore + ) + db_session.add( + Granule( + id="bar", + filename="bar.tif", + tileid="bar", + size=100, + beginposition=ingestion_datetime, + endposition=ingestion_datetime, + ingestiondate=ingestion_datetime, + download_url="https://example.com/bar.tif", + downloaded=False, + ), # type: ignore + ) + db_session.commit() + + actual = _handler( + dict(dry_run=False, date=f"{ingestion_date}"), + lambda: db_session, + sqs_client, + sqs_queue.url, + ) + expected = Response( + dry_run=False, + queue_url=sqs_queue.url, + ingestion_date=f"{ingestion_date}", + count=2, + granules=( + { + "id": "foo", + "filename": "foo.tif", + "download_url": "https://example.com/foo.tif", + }, + { + "id": "bar", + "filename": "bar.tif", + "download_url": "https://example.com/bar.tif", + }, + ), + ) + received = sqs_client.receive_message( + QueueUrl=sqs_queue.url, MaxNumberOfMessages=10 + ) + + assert actual == expected + assert len(received["Messages"]) == 2 + assert received["Messages"][0].get("Body") == json.dumps(expected["granules"][0]) + assert received["Messages"][1].get("Body") == json.dumps(expected["granules"][1])