Update the function app code

.vscode/extensions.json

@@ -0,0 +1,6 @@
+{
+  "recommendations": [
+    "ms-azuretools.vscode-azurefunctions",
+    "ms-python.python"
+  ]
+}

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+  "configurations": [
+    {
+      "connect": {
+        "host": "localhost",
+        "port": 9091
+      },
+      "name": "Attach to Python Functions",
+      "preLaunchTask": "func: host start",
+      "request": "attach",
+      "type": "debugpy"
+    }
+  ],
+  "version": "0.2.0"
+}

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+  "azureFunctions.projectLanguage": "Python",
+  "azureFunctions.projectLanguageModel": 2,
+  "azureFunctions.projectRuntime": "~4",
+  "azureFunctions.scmDoBuildDuringDeployment": true,
+  "debug.internalConsoleOptions": "neverOpen"
+}

.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+  "tasks": [
+    {
+      "command": "host start",
+      "isBackground": true,
+      "label": "func: host start",
+      "options": {
+        "cwd": "${workspaceFolder}/ai_search_with_adi/function_app"
+      },
+      "problemMatcher": "$func-python-watch",
+      "type": "func"
+    }
+  ],
+  "version": "2.0.0"
+}

ai_search_with_adi/ai_search/.env

@@ -4,7 +4,7 @@ FunctionApp__PreEmbeddingCleaner__FunctionName=pre_embedding_cleaner
 FunctionApp__ADI__FunctionName=adi_2_ai_search
 FunctionApp__KeyPhraseExtractor__FunctionName=key_phrase_extractor
 FunctionApp__AppRegistrationResourceId=<App registration in form api://appRegistrationclientId if using identity based connections>
-AIService__AzureSearchOptions__IdentityType=<identityType> # system_assigned or user_assigned or key
+IdentityType=<identityType> # system_assigned or user_assigned or key
 AIService__AzureSearchOptions__Endpoint=<searchServiceEndpoint>
 AIService__AzureSearchOptions__Identity__ClientId=<clientId if using user assigned identity>
 AIService__AzureSearchOptions__Key=<searchServiceKey if not using identity>

ai_search_with_adi/ai_search/ai_search.py

@@ -374,7 +374,7 @@ def get_key_phrase_extraction_skill(self, context, source) -> WebApiSkill:
             InputFieldMappingEntry(name="text", source=source),
         ]
         key_phrase_extraction__skill_outputs = [
-            OutputFieldMappingEntry(name="keyPhrases", target_name="keywords")
+            OutputFieldMappingEntry(name="key_phrases", target_name="keywords")
         ]
         key_phrase_extraction_skill = WebApiSkill(
             name="Key phrase extraction API",

ai_search_with_adi/ai_search/environment.py

@@ -54,7 +54,7 @@ def identity_type(self) -> IdentityType:
         Returns:
             IdentityType: The identity type
         """
-        identity = os.environ.get("AIService__AzureSearchOptions__IdentityType")
+        identity = os.environ.get("IdentityType")
 
         if identity == "user_assigned":
             return IdentityType.USER_ASSIGNED

ai_search_with_adi/ai_search/requirements.txt

@@ -0,0 +1,5 @@
+python-dotenv
+azure-search-documents==11.6.0b4
+azure-storage-blob
+azure-identity
+azure-mgmt-web

ai_search_with_adi/function_app/.env

@@ -0,0 +1,9 @@
+AIService__Services__Endpoint=<AI Services (Cognitive) Endpoint>
+AIService__Services__Key=<AI Services (Cognitive) Key if using key based authentication>
+IdentityType=<identityType> # system_assigned or user_assigned or key
+StorageAccount__ConnectionString=<connectionString if using non managed identity or endpoint if using managed identity>
+OpenAI__ApiKey=<openAIKey if using non managed identity>
+OpenAI__Endpoint=<openAIEndpoint>
+OpenAI__MultiModalDeployment=<openAIEmbeddingDeploymentId>
+OpenAI__ApiVersion=<openAIApiVersion>
+FunctionApp__ClientId=<clientId if using user assigned identity>

ai_search_with_adi/function_app/.funcignore

@@ -0,0 +1,8 @@
+.git*
+.vscode
+__azurite_db*__.json
+__blobstorage__
+__queuestorage__
+local.settings.json
+test
+.venv

ai_search_with_adi/function_app/adi_2_ai_search.py

@@ -1,6 +1,6 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
-
+from azure.identity import DefaultAzureCredential, get_bearer_token_provider
 import base64
 from azure.core.credentials import AzureKeyCredential
 from azure.ai.documentintelligence.aio import DocumentIntelligenceClient
@@ -17,6 +17,7 @@
 import json
 from openai import AsyncAzureOpenAI
 import openai
+from environment import IdentityType, get_identity_type
 
 
 def crop_image_from_pdf_page(pdf_path, page_number, bounding_box):
@@ -134,23 +135,42 @@ async def understand_image_with_gptv(image_base64, caption, tries_left=3):
     """
 
     MAX_TOKENS = 2000
-    api_key = os.environ["AzureAI_GPT4V_Key"]
-    api_version = os.environ["AzureAI__GPT4V_Version"]
-    deployment_name = os.environ["AzureAI__GPT4V_Deployment"]
-    api_base = os.environ["AzureAI__GPT4V_APIbase"]
+    api_version = os.environ.get("OpenAI__ApiVersion")
+    model = os.environ.get("OpenAI__MultiModalDeployment")
+
+    if get_identity_type() != IdentityType.SYSTEM_ASSIGNED:
+        token_provider = get_bearer_token_provider(
+            DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"
+        )
+        api_key = None
+    elif get_identity_type() != IdentityType.USER_ASSIGNED:
+        token_provider = get_bearer_token_provider(
+            DefaultAzureCredential(
+                managed_identity_client_id=os.environ["FunctionApp__ClientId"]
+            ),
+            "https://cognitiveservices.azure.com/.default",
+        )
+        api_key = None
+    else:
+        token_provider = None
+        api_key = os.environ.get("OpenAI__ApiKey")
 
     try:
         async with AsyncAzureOpenAI(
             api_key=api_key,
             api_version=api_version,
-            base_url=f"{api_base}/openai/deployments/{deployment_name}",
+            azure_ad_token_provider=token_provider,
+            azure_endpoint=os.environ.get("OpenAI__AzureEndpoint"),
         ) as client:
             # We send both image caption and the image body to GPTv for better understanding
             if caption != "":
                 response = await client.chat.completions.create(
-                    model=deployment_name,
+                    model=model,
                     messages=[
-                        {"role": "system", "content": "You are a helpful assistant."},
+                        {
+                            "role": "system",
+                            "content": "You are an expert in image analysis. Use your experience and skills to provided a detailed description of any provided images. You should focus on what info can be inferred from the image and the meaning of the data inside the image.",
+                        },
                         {
                             "role": "user",
                             "content": [
@@ -170,9 +190,12 @@ async def understand_image_with_gptv(image_base64, caption, tries_left=3):
 
             else:
                 response = await client.chat.completions.create(
-                    model=deployment_name,
+                    model=model,
                     messages=[
-                        {"role": "system", "content": "You are a helpful assistant."},
+                        {
+                            "role": "system",
+                            "content": "You are an expert in image analysis. Use your experience and skills to provided a detailed description of any provided images. You should focus on what info can be inferred from the image and the meaning of the data inside the image.",
+                        },
                         {
                             "role": "user",
                             "content": [
@@ -323,9 +346,18 @@ async def analyse_document(file_path: str) -> AnalyzeResult:
     with open(file_path, "rb") as f:
         file_read = f.read()
 
+    if get_identity_type() == IdentityType.SYSTEM_ASSIGNED:
+        credential = DefaultAzureCredential()
+    elif get_identity_type() == IdentityType.USER_ASSIGNED:
+        credential = DefaultAzureCredential(
+            managed_identity_client_id=os.environ["FunctionApp__ClientId"]
+        )
+    else:
+        credential = AzureKeyCredential(os.environ["AIService__Services__Key"])
+
     async with DocumentIntelligenceClient(
         endpoint=os.environ["AIService__Services__Endpoint"],
-        credential=AzureKeyCredential(os.environ["AIService__Services__Key"]),
+        credential=credential,
     ) as document_intelligence_client:
         poller = await document_intelligence_client.begin_analyze_document(
             model_id="prebuilt-layout",

ai_search_with_adi/function_app/environment.py

@@ -0,0 +1,30 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+import os
+from enum import Enum
+
+
+class IdentityType(Enum):
+    """The type of the indexer"""
+
+    USER_ASSIGNED = "user_assigned"
+    SYSTEM_ASSIGNED = "system_assigned"
+    KEY = "key"
+
+
+def get_identity_type() -> IdentityType:
+    """This function returns the identity type.
+
+    Returns:
+        IdentityType: The identity type
+    """
+    identity = os.environ.get("IdentityType")
+
+    if identity == "user_assigned":
+        return IdentityType.USER_ASSIGNED
+    elif identity == "system_assigned":
+        return IdentityType.SYSTEM_ASSIGNED
+    elif identity == "key":
+        return IdentityType.KEY
+    else:
+        raise ValueError("Invalid identity type")

ai_search_with_adi/function_app/function_app.py

@@ -7,11 +7,8 @@
 
 from adi_2_ai_search import process_adi_2_ai_search
 from pre_embedding_cleaner import process_pre_embedding_cleaner
-
-
 from key_phrase_extraction import process_key_phrase_extraction
 
-
 logging.basicConfig(level=logging.INFO)
 app = func.FunctionApp(http_auth_level=func.AuthLevel.FUNCTION)
 
@@ -121,8 +118,9 @@ async def key_phrase_extractor(req: func.HttpRequest) -> func.HttpResponse:
 
         results = await asyncio.gather(*record_tasks)
         logging.debug("Results: %s", results)
-        cleaned_tasks = {"values": results}
 
         return func.HttpResponse(
-            json.dumps(cleaned_tasks), status_code=200, mimetype="application/json"
+            json.dumps({"values": results}),
+            status_code=200,
+            mimetype="application/json",
         )

ai_search_with_adi/function_app/host.json

@@ -0,0 +1,16 @@
+{
+  "extensionBundle": {
+    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+    "version": "[4.*, 5.0.0)"
+  },
+  "functionTimeout": "00:05:00",
+  "logging": {
+    "applicationInsights": {
+      "samplingSettings": {
+        "excludedTypes": "Request",
+        "isEnabled": true
+      }
+    }
+  },
+  "version": "2.0"
+}