feat: make sharepoint documents and sharepoint pages optional

Author: Nils Kleinrahm

backend/onyx/connectors/sharepoint/connector.py

@@ -294,6 +294,7 @@ def __init__(
         batch_size: int = INDEX_BATCH_SIZE,
         sites: list[str] = [],
         include_site_pages: bool = True,
+        include_site_documents: bool = True,
     ) -> None:
         self.batch_size = batch_size
         self._graph_client: GraphClient | None = None
@@ -302,6 +303,7 @@ def __init__(
         )
         self.msal_app: msal.ConfidentialClientApplication | None = None
         self.include_site_pages = include_site_pages
+        self.include_site_documents = include_site_documents
 
     @property
     def graph_client(self) -> GraphClient:
@@ -558,18 +560,21 @@ def _fetch_from_sharepoint(
         doc_batch: list[Document] = []
         for site_descriptor in site_descriptors:
             # Fetch regular documents from document libraries
-            driveitems = self._fetch_driveitems(site_descriptor, start=start, end=end)
-            for driveitem, drive_name in driveitems:
-                logger.debug(f"Processing: {driveitem.web_url}")
-
-                # Convert driveitem to document with size checking
-                doc = _convert_driveitem_to_document(driveitem, drive_name)
-                if doc is not None:
-                    doc_batch.append(doc)
-
-                if len(doc_batch) >= self.batch_size:
-                    yield doc_batch
-                    doc_batch = []
+            if self.include_site_documents:
+                driveitems = self._fetch_driveitems(
+                    site_descriptor, start=start, end=end
+                )
+                for driveitem, drive_name in driveitems:
+                    logger.debug(f"Processing: {driveitem.web_url}")
+
+                    # Convert driveitem to document with size checking
+                    doc = _convert_driveitem_to_document(driveitem, drive_name)
+                    if doc is not None:
+                        doc_batch.append(doc)
+
+                    if len(doc_batch) >= self.batch_size:
+                        yield doc_batch
+                        doc_batch = []
 
             # Fetch SharePoint site pages (.aspx files)
             # Only fetch site pages if a folder is not specified since this processing

backend/tests/daily/connectors/sharepoint/test_sharepoint_connector.py

@@ -90,7 +90,9 @@ def test_sharepoint_connector_all_sites__docs_only(
     sharepoint_credentials: dict[str, str],
 ) -> None:
     # Initialize connector with no sites
-    connector = SharepointConnector(include_site_pages=False)
+    connector = SharepointConnector(
+        include_site_pages=False, include_site_documents=True
+    )
 
     # Load credentials
     connector.load_credentials(sharepoint_credentials)
@@ -141,7 +143,9 @@ def test_sharepoint_connector_root_folder__docs_only(
 ) -> None:
     # Initialize connector with the base site URL
     connector = SharepointConnector(
-        sites=[os.environ["SHAREPOINT_SITE"]], include_site_pages=False
+        sites=[os.environ["SHAREPOINT_SITE"]],
+        include_site_pages=False,
+        include_site_documents=True,
     )
 
     # Load credentials

web/src/lib/connectors/connectors.tsx

@@ -666,14 +666,31 @@ See our docs for more details.`,
         name: "sites",
         optional: true,
         description: `• If no sites are specified, all sites in your organization will be indexed (Sites.Read.All permission required).
-
-• Specifying 'https://onyxai.sharepoint.com/sites/support' for example will only index documents within this site.
-
-• Specifying 'https://onyxai.sharepoint.com/sites/support/subfolder' for example will only index documents within this folder.
+• Specifying 'https://onyxai.sharepoint.com/sites/support' for example only indexes this site.
+• Specifying 'https://onyxai.sharepoint.com/sites/support/subfolder' for example only indexes this folder.
 `,
       },
     ],
-    advanced_values: [],
+    advanced_values: [
+      {
+        type: "checkbox",
+        query: "Index Documents:",
+        label: "Index Documents",
+        name: "include_site_documents",
+        optional: true,
+        default: true,
+        description: "Index documents from SharePoint document libraries",
+      },
+      {
+        type: "checkbox",
+        query: "Index ASPX Sites:",
+        label: "Index ASPX Sites",
+        name: "include_site_pages",
+        optional: true,
+        default: true,
+        description: "Index SharePoint site pages (.aspx files)",
+      },
+    ],
   },
   teams: {
     description: "Configure Teams connector",
@@ -1579,6 +1596,8 @@ export interface SalesforceConfig {
 
 export interface SharepointConfig {
   sites?: string[];
+  include_site_pages?: boolean;
+  include_site_documents?: boolean;
 }
 
 export interface TeamsConfig {