From 4d10a601da62ed832080168e22d61f092f92420c Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Tue, 21 Jan 2025 15:42:57 +0000 Subject: [PATCH 1/6] Add example env --- text_2_sql/.env.example | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 text_2_sql/.env.example diff --git a/text_2_sql/.env.example b/text_2_sql/.env.example new file mode 100644 index 0000000..f08cf6d --- /dev/null +++ b/text_2_sql/.env.example @@ -0,0 +1,36 @@ +# Environment variables for Text2SQL +IdentityType= # system_assigned or user_assigned or key + +# Open AI Connection Details +OpenAI__CompletionDeployment= +OpenAI__MiniCompletionDeployment= +OpenAI__Endpoint= +OpenAI__ApiKey= +OpenAI__ApiVersion= + +# Azure AI Search Connection Details +AIService__AzureSearchOptions__Endpoint= +AIService__AzureSearchOptions__Key= +AIService__AzureSearchOptions__Text2SqlSchemaStore__Index= +AIService__AzureSearchOptions__Text2SqlSchemaStore__SemanticConfig= +AIService__AzureSearchOptions__Text2SqlQueryCache__Index= +AIService__AzureSearchOptions__Text2SqlQueryCache__SemanticConfig= +AIService__AzureSearchOptions__Text2SqlColumnValueStore__Index= + +# All SQL Engine specific connection details +Text2Sql__DatabaseName= + +# TSQL or PostgreSQL Specific Connection Details +Text2Sql__DatabaseConnectionString= + +# Snowflake Specific Connection Details +Text2Sql__Snowflake__User= +Text2Sql__Snowflake__Password= +Text2Sql__Snowflake__Account= +Text2Sql__Snowflake__Warehouse= + +# Databricks Specific Connection Details +Text2Sql__Databricks__Catalog= +Text2Sql__Databricks__ServerHostname= +Text2Sql__Databricks__HttpPath= +Text2Sql__Databricks__AccessToken= From 8e72c8f3bd00e4c53a8e51d90b466bbf9edfe93d Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Tue, 21 Jan 2025 16:03:58 +0000 Subject: [PATCH 2/6] Delete some envs --- text_2_sql/data_dictionary/.env | 17 ----------- .../src/text_2_sql_core/connectors/open_ai.py | 30 +++++++++---------- 2 files changed, 15 insertions(+), 32 deletions(-) delete mode 100644 text_2_sql/data_dictionary/.env diff --git a/text_2_sql/data_dictionary/.env b/text_2_sql/data_dictionary/.env deleted file mode 100644 index ad420ec..0000000 --- a/text_2_sql/data_dictionary/.env +++ /dev/null @@ -1,17 +0,0 @@ -OpenAI__CompletionDeployment= -OpenAI__EmbeddingModel= -OpenAI__Endpoint= -OpenAI__ApiKey= -OpenAI__ApiVersion= -Text2Sql__DatabaseName= -Text2Sql__DatabaseConnectionString= -Text2Sql__Snowflake__User= -Text2Sql__Snowflake__Password= -Text2Sql__Snowflake__Account= -Text2Sql__Snowflake__Warehouse= -Text2Sql__Databricks__Catalog= -Text2Sql__Databricks__ServerHostname= -Text2Sql__Databricks__HttpPath= -Text2Sql__Databricks__AccessToken= -IdentityType= # system_assigned or user_assigned or key -ClientId= diff --git a/text_2_sql/text_2_sql_core/src/text_2_sql_core/connectors/open_ai.py b/text_2_sql/text_2_sql_core/src/text_2_sql_core/connectors/open_ai.py index 707daae..7a56542 100644 --- a/text_2_sql/text_2_sql_core/src/text_2_sql_core/connectors/open_ai.py +++ b/text_2_sql/text_2_sql_core/src/text_2_sql_core/connectors/open_ai.py @@ -74,20 +74,20 @@ async def run_completion_request( else: return message.content - async def run_embedding_request(self, batch: list[str]): - token_provider, api_key = self.get_authentication_properties() + # async def run_embedding_request(self, batch: list[str]): + # token_provider, api_key = self.get_authentication_properties() - model_deployment = os.environ["OpenAI__EmbeddingModel"] - async with AsyncAzureOpenAI( - azure_deployment=model_deployment, - api_version=os.environ["OpenAI__ApiVersion"], - azure_endpoint=os.environ["OpenAI__Endpoint"], - azure_ad_token_provider=token_provider, - api_key=api_key, - ) as open_ai_client: - embeddings = await open_ai_client.embeddings.create( - model=os.environ["OpenAI__EmbeddingModel"], - input=batch, - ) + # model_deployment = os.environ["OpenAI__EmbeddingModel"] + # async with AsyncAzureOpenAI( + # azure_deployment=model_deployment, + # api_version=os.environ["OpenAI__ApiVersion"], + # azure_endpoint=os.environ["OpenAI__Endpoint"], + # azure_ad_token_provider=token_provider, + # api_key=api_key, + # ) as open_ai_client: + # embeddings = await open_ai_client.embeddings.create( + # model=os.environ["OpenAI__EmbeddingModel"], + # input=batch, + # ) - return embeddings + # return embeddings From e4f8ab605200c747797212b3fc4d19c1bb593850 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Tue, 21 Jan 2025 16:08:12 +0000 Subject: [PATCH 3/6] Update getting started --- text_2_sql/GETTING_STARTED.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/text_2_sql/GETTING_STARTED.md b/text_2_sql/GETTING_STARTED.md index a39db01..6d6dd83 100644 --- a/text_2_sql/GETTING_STARTED.md +++ b/text_2_sql/GETTING_STARTED.md @@ -5,7 +5,7 @@ To get started, perform the following steps: 1. Setup Azure OpenAI in your subscription with **gpt-4o-mini** & an embedding model, alongside a SQL Server sample database, AI Search and a storage account. 2. Clone this repository and deploy the AI Search text2sql indexes from `deploy_ai_search`. 3. Run `uv sync` within the text_2_sql directory to install dependencies. -4. Configure the .env file based on the provided sample +4. Create your `.env` file based on the provided sample `.env.example`. Place this file in the same place as the `.env.example`. 5. Generate a data dictionary for your target server using the instructions in `data_dictionary`. -6. Upload these data dictionaries to the relevant contains in your storage account. Wait for them to be automatically indexed. +6. Upload these data dictionaries to the relevant containers in your storage account. Wait for them to be automatically indexed with the included skillsets. 7. Navigate to `autogen` directory to view the AutoGen implementation. Follow the steps in `Iteration 5 - Agentic Vector Based Text2SQL.ipynb` to get started. From 22e636ecadbe42ebbb74f32b36ca84b50b3a44fd Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Tue, 21 Jan 2025 16:09:32 +0000 Subject: [PATCH 4/6] Update getting started --- text_2_sql/GETTING_STARTED.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/text_2_sql/GETTING_STARTED.md b/text_2_sql/GETTING_STARTED.md index 6d6dd83..fb95c52 100644 --- a/text_2_sql/GETTING_STARTED.md +++ b/text_2_sql/GETTING_STARTED.md @@ -6,6 +6,6 @@ To get started, perform the following steps: 2. Clone this repository and deploy the AI Search text2sql indexes from `deploy_ai_search`. 3. Run `uv sync` within the text_2_sql directory to install dependencies. 4. Create your `.env` file based on the provided sample `.env.example`. Place this file in the same place as the `.env.example`. -5. Generate a data dictionary for your target server using the instructions in `data_dictionary`. +5. Generate a data dictionary for your target server using the instructions in the **Running** section of the `data_dictionary/README.md`. 6. Upload these data dictionaries to the relevant containers in your storage account. Wait for them to be automatically indexed with the included skillsets. 7. Navigate to `autogen` directory to view the AutoGen implementation. Follow the steps in `Iteration 5 - Agentic Vector Based Text2SQL.ipynb` to get started. From 1b4333b0c4b00bede8583d2204776467a382e593 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Tue, 21 Jan 2025 16:31:26 +0000 Subject: [PATCH 5/6] Update --- text_2_sql/autogen/pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/text_2_sql/autogen/pyproject.toml b/text_2_sql/autogen/pyproject.toml index 3f4c1c0..bd680e1 100644 --- a/text_2_sql/autogen/pyproject.toml +++ b/text_2_sql/autogen/pyproject.toml @@ -41,3 +41,6 @@ databricks = [ postgresql = [ "text_2_sql_core[postgresql]", ] +sqlite = [ + "text_2_sql_core[sqlite]", +] From 2d50f4e82a9420be72dc266c16e4140b34614214 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Tue, 21 Jan 2025 16:41:35 +0000 Subject: [PATCH 6/6] Update readme --- text_2_sql/README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/text_2_sql/README.md b/text_2_sql/README.md index 56afde0..7004d35 100644 --- a/text_2_sql/README.md +++ b/text_2_sql/README.md @@ -54,7 +54,20 @@ As the query cache is shared between users (no data is stored in the cache), a n ![Vector Based with Query Cache Logical Flow.](./images/Agentic%20Text2SQL%20Query%20Cache.png "Agentic Vector Based with Query Cache Logical Flow") -#### Parallel execution +## Agents + +This agentic system contains the following agents: + +- **Query Cache Agent:** Responsible for checking the cache for previously asked questions. +- **Query Decomposition Agent:** Responsible for decomposing complex questions, into sub questions that can be answered with SQL. +- **Schema Selection Agent:** Responsible for extracting key terms from the question and checking the index store for the queries. +- **SQL Query Generation Agent:** Responsible for using the previously extracted schemas and generated SQL queries to answer the question. This agent can request more schemas if needed. This agent will run the query. +- **SQL Query Verification Agent:** Responsible for verifying that the SQL query and results question will answer the question. +- **Answer Generation Agent:** Responsible for taking the database results and generating the final answer for the user. + +The combination of this agent allows the system to answer complex questions, whilst staying under the token limits when including the database schemas. The query cache ensures that previously asked questions, can be answered quickly to avoid degrading user experience. + +### Parallel execution After the first agent has rewritten and decomposed the user input, we execute each of the individual questions in parallel for the quickest time to generate an answer. @@ -189,22 +202,9 @@ Below is a sample entry for a view / table that we which to expose to the LLM. T } ``` -See `./data_dictionary` for more details on how the data dictionary is structured and ways to **automatically generate it**. - -## Agentic Vector Based Approach (Iteration 5) - -This approach builds on the the Vector Based SQL Plugin approach that was previously developed, but adds a agentic approach to the solution. - -This agentic system contains the following agents: - -- **Query Cache Agent:** Responsible for checking the cache for previously asked questions. -- **Query Decomposition Agent:** Responsible for decomposing complex questions, into sub questions that can be answered with SQL. -- **Schema Selection Agent:** Responsible for extracting key terms from the question and checking the index store for the queries. -- **SQL Query Generation Agent:** Responsible for using the previously extracted schemas and generated SQL queries to answer the question. This agent can request more schemas if needed. This agent will run the query. -- **SQL Query Verification Agent:** Responsible for verifying that the SQL query and results question will answer the question. -- **Answer Generation Agent:** Responsible for taking the database results and generating the final answer for the user. - -The combination of this agent allows the system to answer complex questions, whilst staying under the token limits when including the database schemas. The query cache ensures that previously asked questions, can be answered quickly to avoid degrading user experience. +> [!NOTE] +> +> - See `./data_dictionary` for more details on how the data dictionary is structured and ways to **automatically generate it**. ## Tips for good Text2SQL performance.