Merge branch 'main' into marrobi/issue3761

CHANGELOG.md

@@ -6,13 +6,51 @@ FEATURES:
 ENHANCEMENTS:
 
 BUG FIXES:
+* Enabling support for more than 20 users/groups in Workspace API ([#3759](https://github.com/microsoft/AzureTRE/pull/3759  ))
+
+COMPONENTS:
+
+## 0.15.2 (October 24, 2023)
+
+BUG FIXES:
+* Remove .sh extension from nexus renewal script so CRON job executes ([#3742](https://github.com/microsoft/AzureTRE/issues/3742))
 * Upgrade porter version to v1.0.15 and on error getting porter outputs return dict ([#3744](https://github.com/microsoft/AzureTRE/issues/3744))
 * Fix notifications displaying workspace name rather than actual resource ([#3746](https://github.com/microsoft/AzureTRE/issues/3746))
 * Fix SecuredByRole fails if app roles are not loaded  ([#3752](https://github.com/microsoft/AzureTRE/issues/3752))
 * Fix workspace not loading fails if operation or history roles are not loaded  ([#3755](https://github.com/microsoft/AzureTRE/issues/3755))
 
-
 COMPONENTS:
+| name | version |
+| ----- | ----- |
+| devops | 0.5.1 |
+| core | 0.8.9 |
+| ui | 0.5.15 |
+| tre-workspace-base | 1.5.0 |
+| tre-workspace-unrestricted | 0.11.1 |
+| tre-workspace-airlock-import-review | 0.12.7 |
+| tre-service-mlflow | 0.7.7 |
+| tre-workspace-service-health | 0.2.5 |
+| tre-service-databricks | 1.0.3 |
+| tre-service-innereye | 0.6.4 |
+| tre-workspace-service-gitea | 0.8.7 |
+| tre-workspace-service-mysql | 0.4.5 |
+| tre-workspace-service-ohdsi | 0.2.4 |
+| tre-service-guacamole-linuxvm | 0.6.9 |
+| tre-service-guacamole-export-reviewvm | 0.1.8 |
+| tre-service-guacamole-windowsvm | 0.7.9 |
+| tre-service-guacamole-import-reviewvm | 0.2.8 |
+| tre-service-guacamole | 0.10.5 |
+| tre-user-resource-aml-compute-instance | 0.5.7 |
+| tre-service-azureml | 0.8.10 |
+| tre-shared-service-cyclecloud | 0.5.5 |
+| tre-shared-service-databricks-private-auth | 0.1.5 |
+| tre-shared-service-gitea | 0.6.10 |
+| tre-shared-service-airlock-notifier | 0.9.0 |
+| tre-shared-service-admin-vm | 0.4.3 |
+| tre-shared-service-certs | 0.5.1 |
+| tre-shared-service-sonatype-nexus | 2.8.13 |
+| tre-shared-service-firewall | 1.1.5 |
+
 
 ## 0.15.1 (October 12, 2023)
 

api_app/_version.py

@@ -1 +1 @@
-__version__ = "0.15.19"
+__version__ = "0.15.20"

api_app/services/aad_authentication.py

@@ -184,14 +184,13 @@ def _get_token_key(self, key_id: str) -> str:
                         n = int.from_bytes(base64.urlsafe_b64decode(self._ensure_b64padding(key['n'])), "big")
                         e = int.from_bytes(base64.urlsafe_b64decode(self._ensure_b64padding(key['e'])), "big")
                         pub_key = rsa.RSAPublicNumbers(e, n).public_key(default_backend())
-                        pub_key_pkcs1 = pub_key.public_bytes(
+
+                        # Cache the PEM formatted public key.
+                        AzureADAuthorization._jwt_keys[key['kid']] = pub_key.public_bytes(
                             encoding=serialization.Encoding.PEM,
                             format=serialization.PublicFormat.PKCS1
                         )
 
-                        # Cache the PEM formatted public key.
-                        AzureADAuthorization._jwt_keys[key['kid']] = pub_key_pkcs1
-
         return AzureADAuthorization._jwt_keys[key_id]
 
     # The below functions are needed to list which workspaces a specific user has access to i.e. GET /workspaces.
@@ -254,7 +253,18 @@ def _get_user_emails(self, roles_graph_data, msgraph_token):
         batch_request_body = self._get_batch_users_by_role_assignments_body(roles_graph_data)
         headers = self._get_auth_header(msgraph_token)
         headers["Content-type"] = "application/json"
-        users_graph_data = requests.post(batch_endpoint, json=batch_request_body, headers=headers).json()
+        max_number_request = 20
+        requests_from_batch = batch_request_body["requests"]
+        # We split the original batch request body in sub-lits with at most max_number_request elements
+        batch_request_body_list = [requests_from_batch[i:i + max_number_request] for i in range(0, len(requests_from_batch), max_number_request)]
+        users_graph_data = {"responses": []}
+
+        # For each sub-list it's required to call the batch endpoint for retrieveing user/group information
+        for request_body_element in batch_request_body_list:
+            batch_request_body_tmp = {"requests": request_body_element}
+            users_graph_data_tmp = requests.post(batch_endpoint, json=batch_request_body_tmp, headers=headers).json()
+            users_graph_data["responses"] = users_graph_data["responses"] + users_graph_data_tmp["responses"]
+
         return users_graph_data
 
     def _get_user_emails_from_response(self, users_graph_data):

api_app/tests_ma/test_services/test_aad_access_service.py

@@ -1,5 +1,5 @@
 import pytest
-from mock import patch
+from mock import call, patch
 
 from models.domain.authentication import User, RoleAssignment
 from models.domain.workspace import Workspace, WorkspaceRole
@@ -554,6 +554,66 @@ def test_get_workspace_role_assignment_details_with_groups_and_users_assigned_re
     assert "[email protected]" in role_assignment_details["WorkspaceOwner"]
 
 
+@patch("services.aad_authentication.AzureADAuthorization._get_auth_header")
+@patch("services.aad_authentication.AzureADAuthorization._get_batch_users_by_role_assignments_body")
+@patch("requests.post")
+def test_get_user_emails_with_batch_of_more_than_20_requests(mock_graph_post, mock_get_batch_users_by_role_assignments_body, mock_headers):
+    # Arrange
+    access_service = AzureADAuthorization()
+    roles_graph_data = [{"id": "role1"}, {"id": "role2"}]
+    msgraph_token = "token"
+    batch_endpoint = access_service._get_batch_endpoint()
+
+    # mock the response of _get_auth_header
+    headers = {"Authorization": f"Bearer {msgraph_token}"}
+    mock_headers.return_value = headers
+    headers["Content-type"] = "application/json"
+
+    # mock the response of the get batch request for 30 users
+    batch_request_body_first_20 = {
+        "requests": [
+            {"id": f"{i}", "method": "GET", "url": f"/users/{i}"} for i in range(20)
+        ]
+    }
+
+    batch_request_body_last_10 = {
+        "requests": [
+            {"id": f"{i}", "method": "GET", "url": f"/users/{i}"} for i in range(20, 30)
+        ]
+    }
+
+    batch_request_body = {
+        "requests": [
+            {"id": f"{i}", "method": "GET", "url": f"/users/{i}"} for i in range(30)
+        ]
+    }
+
+    mock_get_batch_users_by_role_assignments_body.return_value = batch_request_body
+
+    # Mock the response of the post request
+    mock_graph_post_response = {"responses": [{"id": "user1"}, {"id": "user2"}]}
+    mock_graph_post.return_value.json.return_value = mock_graph_post_response
+
+    # Act
+    users_graph_data = access_service._get_user_emails(roles_graph_data, msgraph_token)
+
+    # Assert
+    assert len(users_graph_data["responses"]) == 4
+    calls = [
+        call(
+            f"{batch_endpoint}",
+            json=batch_request_body_first_20,
+            headers=headers
+        ),
+        call(
+            f"{batch_endpoint}",
+            json=batch_request_body_last_10,
+            headers=headers
+        )
+    ]
+    mock_graph_post.assert_has_calls(calls, any_order=True)
+
+
 def get_mock_batch_response(user_principals, group_principals):
     response_body = {"responses": []}
     for user_principal in user_principals:

docs/tre-admins/configure-airlock-review.md

@@ -40,12 +40,4 @@ Once you're done, click Submit.
 
 Verify that the configuration is working by creating Review VMs for existing import export and export requests (configuration is not verified on update).
 
-
-## Troubleshooting
-
-### Users cannot create Review VMs
-
-If a user sees an error when creating Review VMs, this most likely means that the configuration isn't correct.
-Double-check that all GUIDs don't have any symbols missing, and the names of templates are correct.
-
-[![Review VM Error](../assets/using-review-vm-errors.png)](../assets/using-review-vm-errors.png)
+For troubleshooting guidance please review [the airlock troubleshooting FAQ](../troubleshooting-faq/airlock-troubleshooting.md)

docs/troubleshooting-faq/airlock-troubleshooting.md

@@ -0,0 +1,30 @@
+# Airlock Troubleshooting
+
+## Users cannot create Review VMs
+
+If a user sees an error when creating Review VMs, this most likely means that the configuration isn't correct.
+Double-check that all GUIDs don't have any symbols missing, and the names of templates are correct.
+
+[![Review VM Error](../assets/using-review-vm-errors.png)](../assets/using-review-vm-errors.png)
+
+
+## Files do not appear in Review Data folder on the VM
+
+If the Review Data folder is empty, it's likely because the review VM can't connect to the storage account. Import requests must be reviewed using a VM inside the workspace, and export requests must be reviewed using a VM outside the workspace.
+
+For imports ensure that the `airlock-import-review` workspace template is being used and configured in the airlock configuration for the workspace.
+
+
+## Airlock request does not move through the workflow as expected
+
+If the Airlock request does not move through the workflow as expected, it's likely an issue with the Azure Function that processes airlock requests. This function is deployed as part of the TRE, and can be found in the Azure Portal under the name `func-airlock-processor-<tre_id>`.
+
+To troubleshoot, view the function invocations starting with the StatusChangedQueue Trigger, then the other functions as shown in the image below:
+
+[![Function details](../assets/airlock_functions.png)](../assets/airlock_functions.png)
+
+Look for errors in the function invocations in the same time frame that the airlock request was created. Even if the function executed successfully, there may still be errors within the function invocation details. Invocations that take longer can also be a sign of an issue. For example:
+
+[![Functions error](../assets/airlock_functions_error.png)](../assets/airlock_functions_error.png)
+
+If this error should have been handled please create an issue on the GitHub repository for the Azure TRE.

docs/using-tre/terms-definitions.md

@@ -81,5 +81,3 @@ To use a template, and deploy a resource, the template needs to be registered in
     Once a template is registered it can be used multiple times to deploy multiple workspaces, workspace services etc.
 
 If you want to author your own workspace, workspace service, or user resource template, consult the [template authoring guide](../tre-workspace-authors/authoring-workspace-templates.md)
-
-**Thank you for your patience and support!**

mkdocs.yml

@@ -82,11 +82,6 @@ nav:
           - Set up of a Virtual Machine: using-tre/tre-for-research/using-vms.md
           - Importing/exporting data with Airlock: using-tre/tre-for-research/importing-exporting-data-airlock.md
           - Reviewing Airlock Requests: using-tre/tre-for-research/review-airlock-request.md
-          # - Workspaces:
-          #    - using-tre/wks/index.md # Documentation describing what a workspace is
-          #    - Using Workspaces: using-tre/wks/using-wks.md # Interacting with workspaces (via the UI)
-          #    - The Workspace Owner: using-tre/wks/wks-owner.md # Workspace Owners. The concept, and tasks
-          # - FAQ: using-tre/faq.md # FAQ section (to allow easy contribution)
 
   - Templates and Services: # Docs to highlight and illustrate workspaces, workspace services etc
       - Workspaces:
@@ -166,6 +161,7 @@ nav:
           - Checking the Service Bus: troubleshooting-faq/troubleshooting-sb.md
           - Checking Logs in Application Insights: troubleshooting-faq/app-insights-logs.md
           - Troubleshooting the Resource Processor: troubleshooting-faq/troubleshooting-rp.md
+          - Troubleshooting the Airlock: troubleshooting-faq/airlock-troubleshooting.md
           - Manually edit resources in Cosmos DB: troubleshooting-faq/manually-editing-resources.md
           - Troubleshooting cloud-init: troubleshooting-faq/cloud-init.md
 

templates/shared_services/sonatype-nexus-vm/porter.yaml

@@ -1,7 +1,7 @@
 ---
 schemaVersion: 1.0.0
 name: tre-shared-service-sonatype-nexus
-version: 2.8.12
+version: 2.8.13
 description: "A Sonatype Nexus shared service"
 dockerfile: Dockerfile.tmpl
 registry: azuretre

templates/shared_services/sonatype-nexus-vm/terraform/cloud-config.yaml

@@ -45,6 +45,6 @@ runcmd:
   # Reset the admin password of Nexus to the one created by TF and stored in Key Vault
   - bash /etc/nexus-data/scripts/reset_nexus_password.sh "${NEXUS_ADMIN_PASSWORD}"
   # Invoke Nexus SSL configuration (which will also be ran as CRON daily to renew cert)
-  - bash /etc/cron.daily/configure_nexus_ssl.sh
+  - bash /etc/cron.daily/configure_nexus_ssl
   # Configure Nexus repositories
   - bash /etc/nexus-data/scripts/configure_nexus_repos.sh "${NEXUS_ADMIN_PASSWORD}"

templates/shared_services/sonatype-nexus-vm/terraform/vm.tf

@@ -189,7 +189,7 @@ data "template_cloudinit_config" "nexus_config" {
         },
         {
           content     = data.template_file.configure_nexus_ssl.rendered
-          path        = "/etc/cron.daily/configure_nexus_ssl.sh"
+          path        = "/etc/cron.daily/configure_nexus_ssl"
           permissions = "0755"
         },
         {