Upgrade Python packages, switch to FastAPI lifespan async context man…

…ager (#3765)

* Many Python packages are outdated and need updating
Fixes #3764

.devcontainer/Dockerfile

@@ -64,6 +64,8 @@ RUN export PORTER_VERSION=${PORTER_VERSION} \
 ENV PATH ${PORTER_HOME_V1}:$PATH
 
 # Install requirements
+ARG PIP_VERSION=23.3.1
+RUN  pip3 --no-cache-dir install pip==${PIP_VERSION} && pip3 config set global.disable-pip-version-check true
 COPY ["requirements.txt", "/tmp/pip-tmp/" ]
 COPY ["api_app/requirements.txt", "api_app/requirements-dev.txt", "/tmp/pip-tmp/api_app/" ]
 COPY ["resource_processor/vmss_porter/requirements.txt", "/tmp/pip-tmp/resource_processor/vmss_porter/" ]
@@ -73,7 +75,7 @@ COPY ["airlock_processor/requirements.txt", "/tmp/pip-tmp/airlock_processor/"]
 RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt
 
 # Install azure-cli
-ARG AZURE_CLI_VERSION=2.37.0-1~bullseye
+ARG AZURE_CLI_VERSION=2.50.0-1~bullseye
 COPY .devcontainer/scripts/azure-cli.sh /tmp/
 RUN export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \
     && /tmp/azure-cli.sh

CHANGELOG.md

@@ -9,6 +9,7 @@ FEATURES:
 ENHANCEMENTS:
 
 BUG FIXES:
+* Update Python packages, and fix breaking changes ([#3764](https://github.com/microsoft/AzureTRE/issues/3764))
 * Enabling support for more than 20 users/groups in Workspace API ([#3759](https://github.com/microsoft/AzureTRE/pull/3759  ))
 * Airlock Import Review workspace uses dedicated DNS zone to prevent conflict with core ([#3767](https://github.com/microsoft/AzureTRE/pull/3767))
 

airlock_processor/_version.py

@@ -1 +1 @@
-__version__ = "0.6.1"
+__version__ = "0.7.4"

airlock_processor/requirements.txt

@@ -1,8 +1,8 @@
 # Do not include azure-functions-worker as it may conflict with the Azure Functions platform
-azure-core
-azure-functions
-azure-storage-blob
-azure-identity
-azure-mgmt-storage
-azure-mgmt-resource
-pydantic
+azure-core==1.29.5
+azure-functions==1.17.0
+azure-storage-blob==12.19.0
+azure-identity==1.14.1
+azure-mgmt-storage==21.1.0
+azure-mgmt-resource==23.0.1
+pydantic==1.10.13

airlock_processor/shared_code/blob_operations.py

@@ -1,8 +1,8 @@
 import os
-import datetime
 import logging
 import json
 import re
+from datetime import datetime, timedelta
 from typing import Tuple
 
 from azure.core.exceptions import ResourceExistsError
@@ -69,16 +69,18 @@ def copy_data(source_account_name: str, destination_account_name: str, request_i
         logging.error(msg)
         raise NoFilesInRequestException(msg)
 
-    udk = source_blob_service_client.get_user_delegation_key(datetime.datetime.utcnow() - datetime.timedelta(hours=1),
-                                                             datetime.datetime.utcnow() + datetime.timedelta(hours=1))
-
     # token geneation with expiry of 1 hour. since its not shared, we can leave it to expire (no need to track/delete)
     # Remove sas token if not needed: https://github.com/microsoft/AzureTRE/issues/2034
-    sas_token = generate_container_sas(account_name=source_account_name,
-                                       container_name=container_name,
+    start = datetime.utcnow() - timedelta(minutes=15)
+    expiry = datetime.utcnow() + timedelta(hours=1)
+    udk = source_blob_service_client.get_user_delegation_key(key_start_time=start, key_expiry_time=expiry)
+
+    sas_token = generate_container_sas(container_name=container_name,
+                                       account_name=source_account_name,
                                        user_delegation_key=udk,
                                        permission=ContainerSasPermissions(read=True),
-                                       expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=1))
+                                       start=start,
+                                       expiry=expiry)
 
     source_blob = source_container_client.get_blob_client(blob_name)
     source_url = f'{source_blob.url}?{sas_token}'

api_app/Dockerfile

@@ -19,4 +19,4 @@ COPY . /api
 WORKDIR /api
 RUN groupadd -r api && useradd -r -s /bin/false -g api api_user
 USER api_user
-CMD ["gunicorn", "main:app", "--bind", "0.0.0.0:8000", "-k", "uvicorn.workers.UvicornWorker"]
+CMD ["gunicorn", "main:app", "--bind", "0.0.0.0:8000", "-k", "uvicorn.workers.UvicornWorker","--timeout", "60", "--workers", "5"]

api_app/_version.py

@@ -1 +1 @@
-__version__ = "0.15.18"
+__version__ = "0.16.7"

api_app/db/events.py

@@ -3,13 +3,18 @@
 from azure.cosmos.aio import CosmosClient
 
 from api.dependencies.database import get_db_client
+from db.repositories.resources import ResourceRepository
 from core import config
 
 
-async def bootstrap_database(app) -> None:
+async def bootstrap_database(app) -> bool:
     try:
         client: CosmosClient = await get_db_client(app)
         if client:
             await client.create_database_if_not_exists(id=config.STATE_STORE_DATABASE)
+            # Test access to database
+            await ResourceRepository.create(client)
+            return True
     except Exception as e:
         logging.debug(e)
+        return False

api_app/db/repositories/base.py

@@ -30,7 +30,7 @@ async def _get_container(cls, container_name, partition_key_obj) -> ContainerPro
             raise UnableToAccessDatabase
 
     async def query(self, query: str, parameters: Optional[dict] = None):
-        items = self.container.query_items(query=query, parameters=parameters, enable_cross_partition_query=True)
+        items = self.container.query_items(query=query, parameters=parameters)
         return [i async for i in items]
 
     async def read_item_by_id(self, item_id: str) -> dict:

api_app/db/repositories/operations.py

@@ -192,6 +192,6 @@ async def get_operations_by_resource_id(self, resource_id: str) -> List[Operatio
         return parse_obj_as(List[Operation], operations)
 
     async def resource_has_deployed_operation(self, resource_id: str) -> bool:
-        query = self.operations_query() + f' c.resourceId = "{resource_id}" AND c.action = "{RequestAction.Install}" AND c.status = "{Status.Deployed}"'
+        query = self.operations_query() + f' c.resourceId = "{resource_id}" AND ((c.action = "{RequestAction.Install}" AND c.status = "{Status.Deployed}") OR (c.action = "{RequestAction.Upgrade}" AND c.status = "{Status.Updated}"))'
         operations = await self.query(query=query)
         return len(operations) > 0

api_app/main.py

@@ -1,13 +1,13 @@
 import asyncio
 import logging
 from opencensus.ext.azure.trace_exporter import AzureExporter
+import os
 import uvicorn
 
 from fastapi import FastAPI
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi_utils.tasks import repeat_every
-from service_bus.airlock_request_status_update import receive_step_result_message_and_update_status
+from fastapi.concurrency import asynccontextmanager
 
 from services.tracing import RequestTracerMiddleware
 from opencensus.trace.samplers import ProbabilitySampler
@@ -20,9 +20,29 @@
 from api.errors.validation_error import http422_error_handler
 from api.errors.generic_error import generic_error_handler
 from core import config
-from core.events import create_start_app_handler, create_stop_app_handler
+from db.events import bootstrap_database
 from services.logging import initialize_logging, telemetry_processor_callback_function
 from service_bus.deployment_status_updater import DeploymentStatusUpdater
+from service_bus.airlock_request_status_update import AirlockStatusUpdater
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    app.state.cosmos_client = None
+
+    while not await bootstrap_database(app):
+        await asyncio.sleep(5)
+        logging.warning("Database connection could not be established")
+
+    deploymentStatusUpdater = DeploymentStatusUpdater(app)
+    await deploymentStatusUpdater.init_repos()
+
+    airlockStatusUpdater = AirlockStatusUpdater(app)
+    await airlockStatusUpdater.init_repos()
+
+    asyncio.create_task(deploymentStatusUpdater.receive_messages())
+    asyncio.create_task(airlockStatusUpdater.receive_messages())
+    yield
 
 
 def get_application() -> FastAPI:
@@ -33,16 +53,15 @@ def get_application() -> FastAPI:
         version=config.VERSION,
         docs_url=None,
         redoc_url=None,
-        openapi_url=None
+        openapi_url=None,
+        lifespan=lifespan
     )
 
-    application.add_event_handler("startup", create_start_app_handler(application))
-    application.add_event_handler("shutdown", create_stop_app_handler(application))
-
     try:
-        exporter = AzureExporter(sampler=ProbabilitySampler(1.0))
-        exporter.add_telemetry_processor(telemetry_processor_callback_function)
-        application.add_middleware(RequestTracerMiddleware, exporter=exporter)
+        if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"):
+            exporter = AzureExporter(sampler=ProbabilitySampler(1.0))
+            exporter.add_telemetry_processor(telemetry_processor_callback_function)
+            application.add_middleware(RequestTracerMiddleware, exporter=exporter)
     except Exception:
         logging.exception("Failed to add RequestTracerMiddleware")
 
@@ -64,27 +83,12 @@ def get_application() -> FastAPI:
 
 
 if config.DEBUG:
-    initialize_logging(logging.DEBUG)
+    initialize_logging(logging.DEBUG, add_console_handler=True)
 else:
-    initialize_logging(logging.INFO)
+    initialize_logging(logging.INFO, add_console_handler=False)
 
 app = get_application()
 
 
-@app.on_event("startup")
-async def watch_deployment_status() -> None:
-    logging.info("Starting deployment status watcher thread")
-    statusWatcher = DeploymentStatusUpdater(app)
-    await statusWatcher.init_repos()
-    current_event_loop = asyncio.get_event_loop()
-    asyncio.run_coroutine_threadsafe(statusWatcher.receive_messages(), loop=current_event_loop)
-
-
-@app.on_event("startup")
-@repeat_every(seconds=20, wait_first=True, logger=logging.getLogger())
-async def update_airlock_request_status() -> None:
-    await receive_step_result_message_and_update_status(app)
-
-
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=8000, loop="asyncio")

api_app/requirements-dev.txt

@@ -1,6 +1,5 @@
 # Dev requirements
-pytest-asyncio==0.20.3
-asgi-lifespan~=2.0.0
-httpx~=0.23.1
-mock==5.0.0
-pytest==7.2.0
+pytest-asyncio==0.21.1
+httpx==0.25.0
+mock==5.1.0
+pytest==7.4.3

api_app/requirements.txt

@@ -1,25 +1,25 @@
 # API
-azure-core==1.26.1
-aiohttp==3.8.5
-azure-cosmos==4.3.0
-azure-identity==1.12.0
-azure-mgmt-cosmosdb==9.0.0
-azure-mgmt-compute==29.1.0
-azure-mgmt-costmanagement==3.0.0
-azure-storage-blob==12.15.0
-azure-servicebus==7.8.1
-azure-eventgrid==4.9.1
-fastapi[all]==0.95.0
-fastapi-utils==0.2.1
-gunicorn==20.1.0
-jsonschema[format_nongpl]==4.17.1
-msal~=1.20.0
-opencensus-ext-azure==1.1.7
+azure-core==1.29.5
+aiohttp==3.8.6
+azure-cosmos==4.5.1
+azure-identity==1.14.1
+azure-mgmt-cosmosdb==9.3.0
+azure-mgmt-compute==30.3.0
+azure-mgmt-costmanagement==4.0.1
+azure-storage-blob==12.19.0
+azure-servicebus==7.11.3
+azure-eventgrid==4.15.0
+fastapi==0.104.0
+gunicorn==21.2.0
+jsonschema[format_nongpl]==4.19.1
+msal==1.22.0
+opencensus-ext-azure==1.1.11
 opencensus-ext-logging==0.1.1
-PyJWT==2.6.0
-uvicorn[standard]==0.20.0
+PyJWT==2.8.0
+uvicorn[standard]==0.23.2
 semantic-version==2.10.0
-pytz~=2022.7
-python-dateutil~=2.8.2
-azure-mgmt-resource==22.0.0
-pandas==1.5.2
+pytz==2022.7
+python-dateutil==2.8.2
+azure-mgmt-resource==23.0.1
+pandas==2.0.3
+pydantic==1.10.13

api_app/run_tests_and_exit_succesfully.sh

@@ -6,6 +6,6 @@
 rm -f ../test-results/pytest_api*
 mkdir -p ../test-results
 
-if ! pytest --junit-xml ../test-results/pytest_api_unit.xml --ignore e2e_tests; then
+if ! pytest --junit-xml ../test-results/pytest_api_unit.xml --ignore e2e_tests -W ignore::pytest.PytestUnraisableExceptionWarning -W ignore::DeprecationWarning; then
   touch ../test-results/pytest_api_unit_failed
 fi