Send over less data for document sets

backend/onyx/server/documents/models.py

@@ -406,6 +406,26 @@ class ConnectorCredentialPairDescriptor(BaseModel):
     access_type: AccessType
 
 
+class CCPairSummary(BaseModel):
+    """Simplified connector-credential pair information with just essential data"""
+
+    id: int
+    name: str | None
+    source: DocumentSource
+    access_type: AccessType
+
+    @classmethod
+    def from_cc_pair_descriptor(
+        cls, descriptor: ConnectorCredentialPairDescriptor
+    ) -> "CCPairSummary":
+        return cls(
+            id=descriptor.id,
+            name=descriptor.name,
+            source=descriptor.connector.source,
+            access_type=descriptor.access_type,
+        )
+
+
 class RunConnectorRequest(BaseModel):
     connector_id: int
     credential_ids: list[int] | None = None

backend/onyx/server/features/document_set/api.py

@@ -21,6 +21,7 @@
 from onyx.server.features.document_set.models import CheckDocSetPublicResponse
 from onyx.server.features.document_set.models import DocumentSet
 from onyx.server.features.document_set.models import DocumentSetCreationRequest
+from onyx.server.features.document_set.models import DocumentSetSummary
 from onyx.server.features.document_set.models import DocumentSetUpdateRequest
 from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
 from shared_configs.contextvars import get_current_tenant_id
@@ -154,12 +155,13 @@ def list_document_sets_for_user(
     get_editable: bool = Query(
         False, description="If true, return editable document sets"
     ),
-) -> list[DocumentSet]:
+) -> list[DocumentSetSummary]:
+    document_sets = fetch_all_document_sets_for_user(
+        db_session=db_session, user=user, get_editable=get_editable
+    )
     return [
-        DocumentSet.from_model(ds)
-        for ds in fetch_all_document_sets_for_user(
-            db_session=db_session, user=user, get_editable=get_editable
-        )
+        DocumentSetSummary.from_document_set(DocumentSet.from_model(ds))
+        for ds in document_sets
     ]
 
 

backend/onyx/server/features/document_set/models.py

@@ -5,9 +5,11 @@
 from pydantic import Field
 
 from onyx.db.models import DocumentSet as DocumentSetDBModel
+from onyx.server.documents.models import CCPairSummary
 from onyx.server.documents.models import ConnectorCredentialPairDescriptor
 from onyx.server.documents.models import ConnectorSnapshot
 from onyx.server.documents.models import CredentialSnapshot
+from onyx.server.federated.models import FederatedConnectorSummary
 
 
 class FederatedConnectorConfig(BaseModel):
@@ -26,6 +28,48 @@ class FederatedConnectorDescriptor(BaseModel):
     entities: dict[str, Any]
 
 
+class DocumentSetSummary(BaseModel):
+    """Simplified document set model with minimal data for list views"""
+
+    id: int
+    name: str
+    description: str | None
+    cc_pair_summaries: list[CCPairSummary]
+    is_up_to_date: bool
+    is_public: bool
+    users: list[UUID]
+    groups: list[int]
+    federated_connector_summaries: list[FederatedConnectorSummary] = Field(
+        default_factory=list
+    )
+
+    @classmethod
+    def from_document_set(cls, document_set: "DocumentSet") -> "DocumentSetSummary":
+        """Create a summary from a full DocumentSet model"""
+        return cls(
+            id=document_set.id,
+            name=document_set.name,
+            description=document_set.description,
+            cc_pair_summaries=[
+                CCPairSummary.from_cc_pair_descriptor(desc)
+                for desc in document_set.cc_pair_descriptors
+            ],
+            is_up_to_date=document_set.is_up_to_date,
+            is_public=document_set.is_public,
+            users=document_set.users,
+            groups=document_set.groups,
+            federated_connector_summaries=[
+                FederatedConnectorSummary(
+                    id=fc.id,
+                    name=fc.name,
+                    source=fc.source,
+                    entities=fc.entities,
+                )
+                for fc in document_set.federated_connectors
+            ],
+        )
+
+
 class DocumentSetCreationRequest(BaseModel):
     name: str
     description: str

backend/onyx/server/federated/models.py

@@ -65,6 +65,26 @@ class FederatedConnectorDetail(BaseModel):
     document_sets: list[dict[str, Any]] = Field(default_factory=list)
 
 
+class FederatedConnectorSummary(BaseModel):
+    """Simplified federated connector information with just essential data"""
+
+    id: int
+    name: str
+    source: str  # Using str to match FederatedConnectorDescriptor pattern
+    entities: dict[str, Any]
+
+    @classmethod
+    def from_federated_connector_detail(
+        cls, detail: FederatedConnectorDetail, entities: dict[str, Any]
+    ) -> "FederatedConnectorSummary":
+        return cls(
+            id=detail.id,
+            name=detail.name,
+            source=detail.source,
+            entities=entities,
+        )
+
+
 class FederatedConnectorUpdateRequest(BaseModel):
     credentials: FederatedConnectorCredentials | None = None
 

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx

@@ -39,11 +39,12 @@ import {
 import { Separator } from "@/components/ui/separator";
 
 import { CheckFormField } from "@/components/ui/CheckField";
+import { DocumentSetSummary } from "@/lib/types";
 
 export interface SlackChannelConfigFormFieldsProps {
   isUpdate: boolean;
   isDefault: boolean;
-  documentSets: DocumentSet[];
+  documentSets: DocumentSetSummary[];
   searchEnabledAssistants: MinimalPersonaSnapshot[];
   nonSearchAssistants: MinimalPersonaSnapshot[];
   standardAnswerCategoryResponse: StandardAnswerCategoryResponse;
@@ -72,10 +73,37 @@ export function SlackChannelConfigFormFields({
   const [viewSyncEnabledAssistants, setViewSyncEnabledAssistants] =
     useState(false);
 
-  const documentSetContainsSync = (documentSet: DocumentSet) =>
-    documentSet.cc_pair_descriptors.some(
+  // Helper function to check if a document set contains sync connectors
+  const documentSetContainsSync = (
+    documentSet: DocumentSetSummary | DocumentSet
+  ) => {
+    // Check if it's a DocumentSetSummary (has cc_pair_summaries)
+    if ("cc_pair_summaries" in documentSet) {
+      return documentSet.cc_pair_summaries.some(
+        (summary) => summary.access_type === "sync"
+      );
+    }
+    // Otherwise it's a DocumentSet (has cc_pair_descriptors)
+    return documentSet.cc_pair_descriptors.some(
       (descriptor) => descriptor.access_type === "sync"
     );
+  };
+
+  // Helper function to check if a document set contains private connectors
+  const documentSetContainsPrivate = (
+    documentSet: DocumentSetSummary | DocumentSet
+  ) => {
+    // Check if it's a DocumentSetSummary (has cc_pair_summaries)
+    if ("cc_pair_summaries" in documentSet) {
+      return documentSet.cc_pair_summaries.some(
+        (summary) => summary.access_type === "private"
+      );
+    }
+    // Otherwise it's a DocumentSet (has cc_pair_descriptors)
+    return documentSet.cc_pair_descriptors.some(
+      (descriptor) => descriptor.access_type === "private"
+    );
+  };
 
   const [syncEnabledAssistants, availableAssistants] = useMemo(() => {
     const sync: MinimalPersonaSnapshot[] = [];
@@ -95,20 +123,19 @@ export function SlackChannelConfigFormFields({
 
   const unselectableSets = useMemo(() => {
     return documentSets.filter((ds) =>
-      ds.cc_pair_descriptors.some(
-        (descriptor) => descriptor.access_type === "sync"
-      )
+      ds.cc_pair_summaries.some((summary) => summary.access_type === "sync")
     );
   }, [documentSets]);
+
   const memoizedPrivateConnectors = useMemo(() => {
     const uniqueDescriptors = new Map();
     documentSets.forEach((ds) => {
-      ds.cc_pair_descriptors.forEach((descriptor) => {
+      ds.cc_pair_summaries.forEach((summary) => {
         if (
-          descriptor.access_type === "private" &&
-          !uniqueDescriptors.has(descriptor.id)
+          summary.access_type === "private" &&
+          !uniqueDescriptors.has(summary.id)
         ) {
-          uniqueDescriptors.set(descriptor.id, descriptor);
+          uniqueDescriptors.set(summary.id, summary);
         }
       });
     });
@@ -134,12 +161,6 @@ export function SlackChannelConfigFormFields({
     }
   }, [unselectableSets, values.document_sets, setFieldValue, setPopup]);
 
-  const documentSetContainsPrivate = (documentSet: DocumentSet) => {
-    return documentSet.cc_pair_descriptors.some(
-      (descriptor) => descriptor.access_type === "private"
-    );
-  };
-
   const shouldShowPrivacyAlert = useMemo(() => {
     if (values.knowledge_source === "document_sets") {
       const selectedSets = documentSets.filter((ds) =>
@@ -161,9 +182,7 @@ export function SlackChannelConfigFormFields({
   const selectableSets = useMemo(() => {
     return documentSets.filter(
       (ds) =>
-        !ds.cc_pair_descriptors.some(
-          (descriptor) => descriptor.access_type === "sync"
-        )
+        !ds.cc_pair_summaries.some((summary) => summary.access_type === "sync")
     );
   }, [documentSets]);
 

web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx

@@ -10,7 +10,7 @@ import {
 } from "./lib";
 import {
   ConnectorStatus,
-  DocumentSet,
+  DocumentSetSummary,
   UserGroup,
   UserRole,
   FederatedConnectorConfig,
@@ -32,7 +32,7 @@ interface SetCreationPopupProps {
   userGroups: UserGroup[] | undefined;
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;
-  existingDocumentSet?: DocumentSet;
+  existingDocumentSet?: DocumentSetSummary;
 }
 
 export const DocumentSetCreationForm = ({
@@ -61,14 +61,14 @@ export const DocumentSetCreationForm = ({
           name: existingDocumentSet?.name ?? "",
           description: existingDocumentSet?.description ?? "",
           cc_pair_ids:
-            existingDocumentSet?.cc_pair_descriptors.map(
-              (ccPairDescriptor) => ccPairDescriptor.id
+            existingDocumentSet?.cc_pair_summaries.map(
+              (ccPairSummary) => ccPairSummary.id
             ) ?? [],
           is_public: existingDocumentSet?.is_public ?? true,
           users: existingDocumentSet?.users ?? [],
           groups: existingDocumentSet?.groups ?? [],
           federated_connectors:
-            existingDocumentSet?.federated_connectors?.map((fc) => ({
+            existingDocumentSet?.federated_connector_summaries?.map((fc) => ({
               federated_connector_id: fc.id,
               entities: fc.entities,
             })) ?? [],

web/src/app/admin/documents/sets/hooks.tsx

@@ -1,5 +1,5 @@
 import { errorHandlingFetcher } from "@/lib/fetcher";
-import { DocumentSet } from "@/lib/types";
+import { DocumentSetSummary } from "@/lib/types";
 import useSWR, { mutate } from "swr";
 
 const DOCUMENT_SETS_URL = "/api/manage/document-set";
@@ -13,7 +13,7 @@ export function refreshDocumentSets() {
 export function useDocumentSets(getEditable: boolean = false) {
   const url = getEditable ? GET_EDITABLE_DOCUMENT_SETS_URL : DOCUMENT_SETS_URL;
 
-  const swrResponse = useSWR<DocumentSet[]>(url, errorHandlingFetcher, {
+  const swrResponse = useSWR<DocumentSetSummary[]>(url, errorHandlingFetcher, {
     refreshInterval: 5000, // 5 seconds
   });