Update indexer code

Author: BenConstable9

deploy_ai_search/deploy.py

@@ -22,7 +22,9 @@ def deploy_config(arguments: argparse.Namespace):
         )
     elif arguments.index_type == "text_2_sql":
         index_config = Text2SqlAISearch(
-            suffix=arguments.suffix, rebuild=arguments.rebuild
+            suffix=arguments.suffix,
+            rebuild=arguments.rebuild,
+            single_data_dictionary=arguments.single_data_dictionary,
         )
     elif arguments.index_type == "text_2_sql_query_cache":
         index_config = Text2SqlQueryCacheAISearch(
@@ -57,6 +59,12 @@ def deploy_config(arguments: argparse.Namespace):
         required=False,
         help="Whether want to enable chunking by page in adi skill, if no value is passed considered False",
     )
+    parser.add_argument(
+        "--single_data_dictionary",
+        type=bool,
+        required=False,
+        help="Whether or not a single data dictionary file should be uploaded, or one per entity",
+    )
     parser.add_argument(
         "--suffix",
         type=str,

deploy_ai_search/rag_documents.py

@@ -101,15 +101,17 @@ def get_index_fields(self) -> list[SearchableField]:
                 name="Figures",
                 collection=True,
                 fields=[
-                    SimpleField(
+                    SearchableField(
                         name="FigureId",
                         type=SearchFieldDataType.String,
                         collection=True,
+                        searchable=False,
                     ),
-                    SimpleField(
+                    SearchableField(
                         name="FigureUri",
                         type=SearchFieldDataType.String,
                         collection=True,
+                        searchable=False,
                     ),
                 ],
             ),

deploy_ai_search/text_2_sql.py

@@ -18,6 +18,7 @@
     BlobIndexerDataToExtract,
     IndexerExecutionEnvironment,
     BlobIndexerParsingMode,
+    FieldMappingFunction,
 )
 from ai_search import AISearch
 from environment import (
@@ -28,7 +29,12 @@
 class Text2SqlAISearch(AISearch):
     """This class is used to deploy the sql index."""
 
-    def __init__(self, suffix: str | None = None, rebuild: bool | None = False):
+    def __init__(
+        self,
+        suffix: str | None = None,
+        rebuild: bool | None = False,
+        single_data_dictionary: bool | None = False,
+    ):
         """Initialize the Text2SqlAISearch class. This class implements the deployment of the sql index.
 
         Args:
@@ -38,7 +44,10 @@ def __init__(self, suffix: str | None = None, rebuild: bool | None = False):
         self.indexer_type = IndexerType.TEXT_2_SQL
         super().__init__(suffix, rebuild)
 
-        self.parsing_mode = BlobIndexerParsingMode.JSON_ARRAY
+        if single_data_dictionary:
+            self.parsing_mode = BlobIndexerParsingMode.JSON_ARRAY
+        else:
+            self.parsing_mode = BlobIndexerParsingMode.JSON
 
     def get_index_fields(self) -> list[SearchableField]:
         """This function returns the index fields for sql index.
@@ -47,10 +56,15 @@ def get_index_fields(self) -> list[SearchableField]:
             list[SearchableField]: The index fields for sql index"""
 
         fields = [
+            SimpleField(
+                name="Id",
+                type=SearchFieldDataType.String,
+                key=True,
+                analyzer_name="keyword",
+            ),
             SearchableField(
                 name="Entity",
                 type=SearchFieldDataType.String,
-                key=True,
                 analyzer_name="keyword",
             ),
             SearchableField(
@@ -76,15 +90,17 @@ def get_index_fields(self) -> list[SearchableField]:
                     SearchableField(name="Name", type=SearchFieldDataType.String),
                     SearchableField(name="Definition", type=SearchFieldDataType.String),
                     SearchableField(name="Type", type=SearchFieldDataType.String),
-                    SimpleField(
+                    SearchableField(
                         name="AllowedValues",
                         type=SearchFieldDataType.String,
                         collection=True,
+                        searchable=False,
                     ),
-                    SimpleField(
+                    SearchableField(
                         name="SampleValues",
                         type=SearchFieldDataType.String,
                         collection=True,
+                        searchable=False,
                     ),
                 ],
             ),
@@ -190,6 +206,14 @@ def get_indexer(self) -> SearchIndexer:
                 )
             ],
             output_field_mappings=[
+                FieldMapping(
+                    source_field_name="/document/Entity",
+                    target_field_name="Id",
+                    mapping_function=FieldMappingFunction(
+                        name="base64Encode",
+                        parameters={"useHttpServerUtilityUrlTokenEncode": False},
+                    ),
+                ),
                 FieldMapping(
                     source_field_name="/document/Entity", target_field_name="Entity"
                 ),

deploy_ai_search/text_2_sql_query_cache.py

@@ -77,15 +77,17 @@ def get_index_fields(self) -> list[SearchableField]:
                             SearchableField(
                                 name="Type", type=SearchFieldDataType.String
                             ),
-                            SimpleField(
+                            SearchableField(
                                 name="AllowedValues",
                                 type=SearchFieldDataType.String,
                                 collection=True,
+                                searchable=False,
                             ),
-                            SimpleField(
+                            SearchableField(
                                 name="SampleValues",
                                 type=SearchFieldDataType.String,
                                 collection=True,
+                                searchable=False,
                             ),
                         ],
                     ),