Merge branch 'main' into feature/skip-straight-2-query

Author: BenConstable9

adi_function_app/local.settings.json

@@ -0,0 +1,17 @@
+{
+  "IsEncrypted": false,
+  "Values": {
+    "AIService__DocumentIntelligence__Endpoint": "<documentIntelligenceEndpoint>",
+    "AIService__DocumentIntelligence__Key": "<documentIntelligenceKey if not using identity>",
+    "AIService__Language__Endpoint": "<languageEndpoint>",
+    "AIService__Language__Key": "<languageKey if not using identity>",
+    "FunctionApp__ClientId": "<clientId of the function app if using user assigned managed identity>",
+    "IdentityType": "<identityType> # system_assigned or user_assigned or key",
+    "OpenAI__ApiKey": "<openAIKey if using non managed identity>",
+    "OpenAI__ApiVersion": "<openAIApiVersion>",
+    "OpenAI__Endpoint": "<openAIEndpoint>",
+    "OpenAI__MultiModalDeployment": "<openAIEmbeddingDeploymentId>",
+    "StorageAccount__ConnectionString": "<connectionString if using non managed identity>",
+    "StorageAccount__Endpoint": "<Endpoint if using identity based connections>"
+  }
+}

adi_function_app/pre_embedding_cleaner.py

@@ -71,15 +71,21 @@ def clean_text(src_text: str) -> str:
     try:
         # Define specific patterns for each tag
         tag_patterns = {
-            "figurecontent": r"<!-- FigureContent=(.*?)-->",
+            "figurecontent": r"<!--.*?FigureContent=(.*?)-->",
             "figure": r"<figure>(.*?)</figure>",
             "figures": r"\(figures/\d+\)(.*?)\(figures/\d+\)",
             "figcaption": r"<figcaption>(.*?)</figcaption>",
         }
         cleaned_text = remove_markdown_tags(src_text, tag_patterns)
 
-        # remove line breaks
-        cleaned_text = re.sub(r"\n", "", cleaned_text)
+        # remove html tags
+        cleaned_text = re.sub(r"<.*?>", "", cleaned_text)
+
+        # Replace newline characters with spaces
+        cleaned_text = re.sub(r"\n", " ", cleaned_text)
+
+        # Replace multiple whitespace characters with a single space
+        cleaned_text = re.sub(r"\s+", " ", cleaned_text)
 
         # remove stopwords
         tokens = word_tokenize(cleaned_text, "english")

deploy_ai_search/ai_search.py

@@ -14,7 +14,7 @@
     WebApiSkill,
     AzureOpenAIEmbeddingSkill,
     AzureOpenAIVectorizer,
-    AzureOpenAIParameters,
+    AzureOpenAIVectorizerParameters,
     SearchIndexer,
     SearchIndexerSkillset,
     SearchIndexerDataContainer,
@@ -24,7 +24,7 @@
     InputFieldMappingEntry,
     SynonymMap,
     SplitSkill,
-    SearchIndexerIndexProjections,
+    SearchIndexerIndexProjection,
     BlobIndexerParsingMode,
 )
 from azure.core.exceptions import HttpResponseError
@@ -146,7 +146,7 @@ def get_indexer(self) -> SearchIndexer:
 
         return None
 
-    def get_index_projections(self) -> SearchIndexerIndexProjections:
+    def get_index_projections(self) -> SearchIndexerIndexProjection:
         """Get the index projections for the indexer."""
 
         return None
@@ -353,9 +353,9 @@ def get_vector_skill(
             name="Vector Skill",
             description="Skill to generate embeddings",
             context=context,
-            deployment_id=self.environment.open_ai_embedding_deployment,
+            deployment_name=self.environment.open_ai_embedding_deployment,
             model_name=self.environment.open_ai_embedding_model,
-            resource_uri=self.environment.open_ai_endpoint,
+            resource_url=self.environment.open_ai_endpoint,
             inputs=embedding_skill_inputs,
             outputs=embedding_skill_outputs,
             dimensions=self.environment.open_ai_embedding_dimensions,
@@ -430,10 +430,10 @@ def get_vector_search(self) -> VectorSearch:
             VectorSearch: The vector search configuration
         """
 
-        open_ai_params = AzureOpenAIParameters(
-            resource_uri=self.environment.open_ai_endpoint,
+        open_ai_params = AzureOpenAIVectorizerParameters(
+            resource_url=self.environment.open_ai_endpoint,
             model_name=self.environment.open_ai_embedding_model,
-            deployment_id=self.environment.open_ai_embedding_deployment,
+            deployment_name=self.environment.open_ai_embedding_deployment,
         )
 
         if self.environment.identity_type == IdentityType.KEY:
@@ -451,13 +451,13 @@ def get_vector_search(self) -> VectorSearch:
                 VectorSearchProfile(
                     name=self.vector_search_profile_name,
                     algorithm_configuration_name=self.algorithm_name,
-                    vectorizer=self.vectorizer_name,
+                    vectorizer_name=self.vectorizer_name,
                 )
             ],
             vectorizers=[
                 AzureOpenAIVectorizer(
-                    name=self.vectorizer_name,
-                    azure_open_ai_parameters=open_ai_params,
+                    vectorizer_name=self.vectorizer_name,
+                    parameters=open_ai_params,
                 ),
             ],
         )
@@ -497,7 +497,7 @@ def deploy_skillset(self):
             name=self.skillset_name,
             description="Skillset to chunk documents and generating embeddings",
             skills=skills,
-            index_projections=index_projections,
+            index_projection=index_projections,
         )
 
         self._search_indexer_client.create_or_update_skillset(skillset)

deploy_ai_search/environment.py

@@ -95,7 +95,7 @@ def ai_search_user_assigned_identity(self) -> SearchIndexerDataUserAssignedIdent
             SearchIndexerDataUserAssignedIdentity: The ai search user assigned identity
         """
         user_assigned_identity = SearchIndexerDataUserAssignedIdentity(
-            user_assigned_identity=os.environ.get(
+            resource_id=os.environ.get(
                 "AIService__AzureSearchOptions__Identity__FQName"
             )
         )

deploy_ai_search/rag_documents.py

@@ -14,7 +14,7 @@
     FieldMapping,
     IndexingParameters,
     IndexingParametersConfiguration,
-    SearchIndexerIndexProjections,
+    SearchIndexerIndexProjection,
     SearchIndexerIndexProjectionSelector,
     SearchIndexerIndexProjectionsParameters,
     IndexProjectionMode,
@@ -199,7 +199,7 @@ def get_skills(self) -> list:
 
         return skills
 
-    def get_index_projections(self) -> SearchIndexerIndexProjections:
+    def get_index_projections(self) -> SearchIndexerIndexProjection:
         """This function returns the index projections for rag document."""
         mappings = [
             InputFieldMappingEntry(name="Chunk", source="/document/pages/*/chunk"),
@@ -241,7 +241,7 @@ def get_index_projections(self) -> SearchIndexerIndexProjections:
                 ]
             )
 
-        index_projections = SearchIndexerIndexProjections(
+        index_projections = SearchIndexerIndexProjection(
             selectors=[
                 SearchIndexerIndexProjectionSelector(
                     target_index_name=self.index_name,

deploy_ai_search/requirements.txt

@@ -1,5 +1,5 @@
 python-dotenv
-azure-search-documents==11.6.0b4
+azure-search-documents==11.6.0b5
 azure-storage-blob
 azure-identity
 azure-mgmt-web

text_2_sql/requirements.txt

@@ -1,6 +1,6 @@
 semantic-kernel==1.9.0
 azure-search
-azure-search-documents==11.6.0b4
+azure-search-documents==11.6.0b5
 aioodbc
 azure-identity
 python-dotenv