Add last modified date

Author: BenConstable9

deploy_ai_search/rag_documents.py

@@ -96,6 +96,11 @@ def get_index_fields(self) -> list[SearchableField]:
                 filterable=True,
                 facetable=True,
             ),
+            SimpleField(
+                name="DateLastModified",
+                type=SearchFieldDataType.DateTimeOffset,
+                filterable=True,
+            ),
         ]
 
         if self.enable_page_by_chunking:
@@ -193,6 +198,9 @@ def get_index_projections(self) -> SearchIndexerIndexProjections:
             InputFieldMappingEntry(
                 name="Sections", source="/document/pages/*/cleaned_sections"
             ),
+            InputFieldMappingEntry(
+                name="DateLastModified", source="/document/DateLastModified"
+            ),
         ]
 
         if self.enable_page_by_chunking:
@@ -269,6 +277,10 @@ def get_indexer(self) -> SearchIndexer:
                     source_field_name="metadata_storage_path",
                     target_field_name="SourceUri",
                 ),
+                FieldMapping(
+                    source_field_name="metadata_storage_last_modified",
+                    target_field_name="DateLastModified",
+                ),
             ],
             parameters=indexer_parameters,
         )

deploy_ai_search/text_2_sql.py

@@ -94,6 +94,11 @@ def get_index_fields(self) -> list[SearchableField]:
                 collection=True,
                 hidden=True,
             ),  # This is needed to enable semantic searching against the column names as complex field types are not used.
+            SimpleField(
+                name="DateLastModified",
+                type=SearchFieldDataType.DateTimeOffset,
+                filterable=True,
+            ),
         ]
 
         return fields
@@ -177,6 +182,12 @@ def get_indexer(self) -> SearchIndexer:
             target_index_name=self.index_name,
             data_source_name=self.data_source_name,
             schedule=schedule,
+            field_mappings=[
+                FieldMapping(
+                    source_field_name="metadata_storage_last_modified",
+                    target_field_name="DateLastModified",
+                )
+            ],
             output_field_mappings=[
                 FieldMapping(
                     source_field_name="/document/Entity", target_field_name="Entity"
@@ -201,6 +212,9 @@ def get_indexer(self) -> SearchIndexer:
                     source_field_name="/document/Columns/*/Name",
                     target_field_name="ColumnNames",
                 ),
+                FieldMapping(
+                    name="DateLastModified", source="/document/DateLastModified"
+                ),
             ],
             parameters=indexer_parameters,
         )

deploy_ai_search/text_2_sql_query_cache.py

@@ -97,6 +97,11 @@ def get_index_fields(self) -> list[SearchableField]:
                     ),
                 ],
             ),
+            SimpleField(
+                name="DateLastModified",
+                type=SearchFieldDataType.DateTimeOffset,
+                filterable=True,
+            ),
         ]
 
         return fields

text_2_sql/ai_search.py

@@ -9,6 +9,7 @@
 import os
 import logging
 import base64
+from datetime import datetime, timezone
 
 
 async def run_ai_search_query(
@@ -87,6 +88,8 @@ async def add_entry_to_index(document: dict, vector_fields: dict, index_name: st
 
     fields_to_embed = {field: document[field] for field in vector_fields}
 
+    document["DateLastModified"] = datetime.now(timezone.utc)
+
     async with AsyncAzureOpenAI(
         # This is the default and can be omitted
         api_key=os.environ["OpenAI__ApiKey"],