Significant Improvements to Agentic Flow (#91)

Moves code to common core.

Adds column store for filter values
Improves the prompts
Fixes question decomposition agent

Closes #79

Author: BenConstable9

.devcontainer/Dockerfile

@@ -3,12 +3,15 @@ FROM mcr.microsoft.com/devcontainers/python:3
 
 # Install ODBC drivers and dependencies
 RUN apt-get update && \
-    apt-get install -y apt-transport-https curl && \
+    apt-get install -y apt-transport-https curl gnupg && \
     curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
     curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
     apt-get update && \
     ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev
 
+# Install Azure CLI
+RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
 # Set the working directory
 WORKDIR /app
 

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
     rev: v2.1.0
     hooks:
       - id: codespell
-        args: [--exclude-file, uv.lock]
+        args: [--skip, "uv.lock,*.json,*,jsonl"]
 
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 23.12.1
@@ -50,4 +50,4 @@ repos:
     rev: 0.5.5
     hooks:
       # Update the uv lockfile
-      - id: uv-lock
+      - id: uv-lock

deploy_ai_search/src/deploy_ai_search/README.md renamed to deploy_ai_search/README.md

@@ -77,21 +77,29 @@ def get_index_fields(self) -> list[SearchableField]:
                 analyzer_name="keyword",
             ),
             SimpleField(
-                name="Entity",
+                name="FQN",
                 type=SearchFieldDataType.String,
             ),
             SimpleField(
-                name="Database",
+                name="Entity",
                 type=SearchFieldDataType.String,
             ),
             SimpleField(
-                name="Warehouse",
+                name="Schema",
                 type=SearchFieldDataType.String,
             ),
             SimpleField(
                 name="Catalog",
                 type=SearchFieldDataType.String,
             ),
+            SimpleField(
+                name="Database",
+                type=SearchFieldDataType.String,
+            ),
+            SimpleField(
+                name="Warehouse",
+                type=SearchFieldDataType.String,
+            ),
             SimpleField(
                 name="Column",
                 type=SearchFieldDataType.String,
@@ -178,16 +186,25 @@ def get_indexer(self) -> SearchIndexer:
             ],
             output_field_mappings=[
                 FieldMapping(
-                    source_field_name="/document/Id",
+                    source_field_name="/document/FQN",
                     target_field_name="Id",
                     mapping_function=FieldMappingFunction(
                         name="base64Encode",
                         parameters={"useHttpServerUtilityUrlTokenEncode": False},
                     ),
                 ),
+                FieldMapping(
+                    source_field_name="/document/FQN", target_field_name="FQN"
+                ),
                 FieldMapping(
                     source_field_name="/document/Entity", target_field_name="Entity"
                 ),
+                FieldMapping(
+                    source_field_name="/document/Schema", target_field_name="Schema"
+                ),
+                FieldMapping(
+                    source_field_name="/document/Catalog", target_field_name="Catalog"
+                ),
                 FieldMapping(
                     source_field_name="/document/Database",
                     target_field_name="Database",

deploy_ai_search/src/deploy_ai_search/text_2_sql_column_value_store.py

@@ -78,11 +78,6 @@ def get_index_fields(self) -> list[SearchableField]:
                 name="SqlQueryDecomposition",
                 collection=True,
                 fields=[
-                    SearchableField(
-                        name="SubQuestion",
-                        type=SearchFieldDataType.String,
-                        filterable=True,
-                    ),
                     SearchableField(
                         name="SqlQuery",
                         type=SearchFieldDataType.String,
@@ -200,7 +195,7 @@ def get_indexer(self) -> SearchIndexer:
                 fail_on_unprocessable_document=False,
                 fail_on_unsupported_content_type=False,
                 index_storage_metadata_only_for_oversized_documents=True,
-                indexed_file_name_extensions=".json",
+                indexed_file_name_extensions=".jsonl",
                 parsing_mode=self.parsing_mode,
             ),
             max_failed_items=5,

deploy_ai_search/src/deploy_ai_search/text_2_sql_query_cache.py

@@ -85,6 +85,11 @@ def get_index_fields(self) -> list[SearchableField]:
                 key=True,
                 analyzer_name="keyword",
             ),
+            SearchableField(
+                name="FQN",
+                type=SearchFieldDataType.String,
+                analyzer_name="keyword",
+            ),
             SearchableField(
                 name="EntityName", type=SearchFieldDataType.String, filterable=True
             ),
@@ -93,6 +98,11 @@ def get_index_fields(self) -> list[SearchableField]:
                 type=SearchFieldDataType.String,
                 analyzer_name="keyword",
             ),
+            SearchableField(
+                name="Schema",
+                type=SearchFieldDataType.String,
+                analyzer_name="keyword",
+            ),
             SearchableField(
                 name="Database",
                 type=SearchFieldDataType.String,
@@ -279,20 +289,29 @@ def get_indexer(self) -> SearchIndexer:
             ],
             output_field_mappings=[
                 FieldMapping(
-                    source_field_name="/document/Entity",
+                    source_field_name="/document/FQN",
                     target_field_name="Id",
                     mapping_function=FieldMappingFunction(
                         name="base64Encode",
                         parameters={"useHttpServerUtilityUrlTokenEncode": False},
                     ),
                 ),
                 FieldMapping(
-                    source_field_name="/document/Entity", target_field_name="Entity"
+                    source_field_name="/document/FQN", target_field_name="FQN"
                 ),
                 FieldMapping(
                     source_field_name="/document/EntityName",
                     target_field_name="EntityName",
                 ),
+                FieldMapping(
+                    source_field_name="/document/Entity", target_field_name="Entity"
+                ),
+                FieldMapping(
+                    source_field_name="/document/Schema", target_field_name="Schema"
+                ),
+                FieldMapping(
+                    source_field_name="/document/Catalog", target_field_name="Catalog"
+                ),
                 FieldMapping(
                     source_field_name="/document/Database",
                     target_field_name="Database",

deploy_ai_search/src/deploy_ai_search/text_2_sql_schema_store.py

@@ -4,6 +4,9 @@ version = "0.1.0"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.12"
+dependencies = [
+    "text-2-sql-core",
+]
 
 [dependency-groups]
 dev = [
@@ -18,3 +21,6 @@ dev = [
 
 [tool.uv.workspace]
 members = ["text_2_sql/text_2_sql_core", "text_2_sql/autogen", "deploy_ai_search", "adi_function_app", "text_2_sql/semantic_kernel"]
+
+[tool.uv.sources]
+text-2-sql-core = { workspace = true }