From 8e8bc6a0fac479346b3164320e6a53a7f13bb4cb Mon Sep 17 00:00:00 2001 From: Weves Date: Sat, 12 Jul 2025 15:40:50 -0700 Subject: [PATCH 1/5] Send over less data for document sets --- backend/onyx/server/documents/models.py | 20 +++++++ .../onyx/server/features/document_set/api.py | 12 ++-- .../server/features/document_set/models.py | 44 ++++++++++++++ backend/onyx/server/federated/models.py | 20 +++++++ .../channels/SlackChannelConfigFormFields.tsx | 57 ++++++++++++------- .../sets/DocumentSetCreationForm.tsx | 10 ++-- web/src/app/admin/documents/sets/hooks.tsx | 4 +- web/src/app/admin/documents/sets/page.tsx | 50 +++++++++------- .../documentSet/DocumentSetSelectable.tsx | 8 +-- web/src/lib/types.ts | 27 +++++++++ 10 files changed, 195 insertions(+), 57 deletions(-) diff --git a/backend/onyx/server/documents/models.py b/backend/onyx/server/documents/models.py index 49237afaaad..b13f80ffb5c 100644 --- a/backend/onyx/server/documents/models.py +++ b/backend/onyx/server/documents/models.py @@ -406,6 +406,26 @@ class ConnectorCredentialPairDescriptor(BaseModel): access_type: AccessType +class CCPairSummary(BaseModel): + """Simplified connector-credential pair information with just essential data""" + + id: int + name: str | None + source: DocumentSource + access_type: AccessType + + @classmethod + def from_cc_pair_descriptor( + cls, descriptor: ConnectorCredentialPairDescriptor + ) -> "CCPairSummary": + return cls( + id=descriptor.id, + name=descriptor.name, + source=descriptor.connector.source, + access_type=descriptor.access_type, + ) + + class RunConnectorRequest(BaseModel): connector_id: int credential_ids: list[int] | None = None diff --git a/backend/onyx/server/features/document_set/api.py b/backend/onyx/server/features/document_set/api.py index 60e70faad43..34165957137 100644 --- a/backend/onyx/server/features/document_set/api.py +++ b/backend/onyx/server/features/document_set/api.py @@ -21,6 +21,7 @@ from onyx.server.features.document_set.models import CheckDocSetPublicResponse from onyx.server.features.document_set.models import DocumentSet from onyx.server.features.document_set.models import DocumentSetCreationRequest +from onyx.server.features.document_set.models import DocumentSetSummary from onyx.server.features.document_set.models import DocumentSetUpdateRequest from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop from shared_configs.contextvars import get_current_tenant_id @@ -154,12 +155,13 @@ def list_document_sets_for_user( get_editable: bool = Query( False, description="If true, return editable document sets" ), -) -> list[DocumentSet]: +) -> list[DocumentSetSummary]: + document_sets = fetch_all_document_sets_for_user( + db_session=db_session, user=user, get_editable=get_editable + ) return [ - DocumentSet.from_model(ds) - for ds in fetch_all_document_sets_for_user( - db_session=db_session, user=user, get_editable=get_editable - ) + DocumentSetSummary.from_document_set(DocumentSet.from_model(ds)) + for ds in document_sets ] diff --git a/backend/onyx/server/features/document_set/models.py b/backend/onyx/server/features/document_set/models.py index 7f75b5cdae9..410bf0fc091 100644 --- a/backend/onyx/server/features/document_set/models.py +++ b/backend/onyx/server/features/document_set/models.py @@ -5,9 +5,11 @@ from pydantic import Field from onyx.db.models import DocumentSet as DocumentSetDBModel +from onyx.server.documents.models import CCPairSummary from onyx.server.documents.models import ConnectorCredentialPairDescriptor from onyx.server.documents.models import ConnectorSnapshot from onyx.server.documents.models import CredentialSnapshot +from onyx.server.federated.models import FederatedConnectorSummary class FederatedConnectorConfig(BaseModel): @@ -26,6 +28,48 @@ class FederatedConnectorDescriptor(BaseModel): entities: dict[str, Any] +class DocumentSetSummary(BaseModel): + """Simplified document set model with minimal data for list views""" + + id: int + name: str + description: str | None + cc_pair_summaries: list[CCPairSummary] + is_up_to_date: bool + is_public: bool + users: list[UUID] + groups: list[int] + federated_connector_summaries: list[FederatedConnectorSummary] = Field( + default_factory=list + ) + + @classmethod + def from_document_set(cls, document_set: "DocumentSet") -> "DocumentSetSummary": + """Create a summary from a full DocumentSet model""" + return cls( + id=document_set.id, + name=document_set.name, + description=document_set.description, + cc_pair_summaries=[ + CCPairSummary.from_cc_pair_descriptor(desc) + for desc in document_set.cc_pair_descriptors + ], + is_up_to_date=document_set.is_up_to_date, + is_public=document_set.is_public, + users=document_set.users, + groups=document_set.groups, + federated_connector_summaries=[ + FederatedConnectorSummary( + id=fc.id, + name=fc.name, + source=fc.source, + entities=fc.entities, + ) + for fc in document_set.federated_connectors + ], + ) + + class DocumentSetCreationRequest(BaseModel): name: str description: str diff --git a/backend/onyx/server/federated/models.py b/backend/onyx/server/federated/models.py index 3813872faee..95b0748a1c5 100644 --- a/backend/onyx/server/federated/models.py +++ b/backend/onyx/server/federated/models.py @@ -65,6 +65,26 @@ class FederatedConnectorDetail(BaseModel): document_sets: list[dict[str, Any]] = Field(default_factory=list) +class FederatedConnectorSummary(BaseModel): + """Simplified federated connector information with just essential data""" + + id: int + name: str + source: str # Using str to match FederatedConnectorDescriptor pattern + entities: dict[str, Any] + + @classmethod + def from_federated_connector_detail( + cls, detail: FederatedConnectorDetail, entities: dict[str, Any] + ) -> "FederatedConnectorSummary": + return cls( + id=detail.id, + name=detail.name, + source=detail.source, + entities=entities, + ) + + class FederatedConnectorUpdateRequest(BaseModel): credentials: FederatedConnectorCredentials | None = None diff --git a/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx b/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx index eeb701265de..ce0d62807f5 100644 --- a/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx +++ b/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx @@ -39,11 +39,12 @@ import { import { Separator } from "@/components/ui/separator"; import { CheckFormField } from "@/components/ui/CheckField"; +import { DocumentSetSummary } from "@/lib/types"; export interface SlackChannelConfigFormFieldsProps { isUpdate: boolean; isDefault: boolean; - documentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; searchEnabledAssistants: MinimalPersonaSnapshot[]; nonSearchAssistants: MinimalPersonaSnapshot[]; standardAnswerCategoryResponse: StandardAnswerCategoryResponse; @@ -72,10 +73,37 @@ export function SlackChannelConfigFormFields({ const [viewSyncEnabledAssistants, setViewSyncEnabledAssistants] = useState(false); - const documentSetContainsSync = (documentSet: DocumentSet) => - documentSet.cc_pair_descriptors.some( + // Helper function to check if a document set contains sync connectors + const documentSetContainsSync = ( + documentSet: DocumentSetSummary | DocumentSet + ) => { + // Check if it's a DocumentSetSummary (has cc_pair_summaries) + if ("cc_pair_summaries" in documentSet) { + return documentSet.cc_pair_summaries.some( + (summary) => summary.access_type === "sync" + ); + } + // Otherwise it's a DocumentSet (has cc_pair_descriptors) + return documentSet.cc_pair_descriptors.some( (descriptor) => descriptor.access_type === "sync" ); + }; + + // Helper function to check if a document set contains private connectors + const documentSetContainsPrivate = ( + documentSet: DocumentSetSummary | DocumentSet + ) => { + // Check if it's a DocumentSetSummary (has cc_pair_summaries) + if ("cc_pair_summaries" in documentSet) { + return documentSet.cc_pair_summaries.some( + (summary) => summary.access_type === "private" + ); + } + // Otherwise it's a DocumentSet (has cc_pair_descriptors) + return documentSet.cc_pair_descriptors.some( + (descriptor) => descriptor.access_type === "private" + ); + }; const [syncEnabledAssistants, availableAssistants] = useMemo(() => { const sync: MinimalPersonaSnapshot[] = []; @@ -95,20 +123,19 @@ export function SlackChannelConfigFormFields({ const unselectableSets = useMemo(() => { return documentSets.filter((ds) => - ds.cc_pair_descriptors.some( - (descriptor) => descriptor.access_type === "sync" - ) + ds.cc_pair_summaries.some((summary) => summary.access_type === "sync") ); }, [documentSets]); + const memoizedPrivateConnectors = useMemo(() => { const uniqueDescriptors = new Map(); documentSets.forEach((ds) => { - ds.cc_pair_descriptors.forEach((descriptor) => { + ds.cc_pair_summaries.forEach((summary) => { if ( - descriptor.access_type === "private" && - !uniqueDescriptors.has(descriptor.id) + summary.access_type === "private" && + !uniqueDescriptors.has(summary.id) ) { - uniqueDescriptors.set(descriptor.id, descriptor); + uniqueDescriptors.set(summary.id, summary); } }); }); @@ -134,12 +161,6 @@ export function SlackChannelConfigFormFields({ } }, [unselectableSets, values.document_sets, setFieldValue, setPopup]); - const documentSetContainsPrivate = (documentSet: DocumentSet) => { - return documentSet.cc_pair_descriptors.some( - (descriptor) => descriptor.access_type === "private" - ); - }; - const shouldShowPrivacyAlert = useMemo(() => { if (values.knowledge_source === "document_sets") { const selectedSets = documentSets.filter((ds) => @@ -161,9 +182,7 @@ export function SlackChannelConfigFormFields({ const selectableSets = useMemo(() => { return documentSets.filter( (ds) => - !ds.cc_pair_descriptors.some( - (descriptor) => descriptor.access_type === "sync" - ) + !ds.cc_pair_summaries.some((summary) => summary.access_type === "sync") ); }, [documentSets]); diff --git a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx index 3a016c24803..fee2ad32ed6 100644 --- a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx +++ b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx @@ -10,7 +10,7 @@ import { } from "./lib"; import { ConnectorStatus, - DocumentSet, + DocumentSetSummary, UserGroup, UserRole, FederatedConnectorConfig, @@ -32,7 +32,7 @@ interface SetCreationPopupProps { userGroups: UserGroup[] | undefined; onClose: () => void; setPopup: (popupSpec: PopupSpec | null) => void; - existingDocumentSet?: DocumentSet; + existingDocumentSet?: DocumentSetSummary; } export const DocumentSetCreationForm = ({ @@ -61,14 +61,14 @@ export const DocumentSetCreationForm = ({ name: existingDocumentSet?.name ?? "", description: existingDocumentSet?.description ?? "", cc_pair_ids: - existingDocumentSet?.cc_pair_descriptors.map( - (ccPairDescriptor) => ccPairDescriptor.id + existingDocumentSet?.cc_pair_summaries.map( + (ccPairSummary) => ccPairSummary.id ) ?? [], is_public: existingDocumentSet?.is_public ?? true, users: existingDocumentSet?.users ?? [], groups: existingDocumentSet?.groups ?? [], federated_connectors: - existingDocumentSet?.federated_connectors?.map((fc) => ({ + existingDocumentSet?.federated_connector_summaries?.map((fc) => ({ federated_connector_id: fc.id, entities: fc.entities, })) ?? [], diff --git a/web/src/app/admin/documents/sets/hooks.tsx b/web/src/app/admin/documents/sets/hooks.tsx index 1362cfda23e..f13761a0ebe 100644 --- a/web/src/app/admin/documents/sets/hooks.tsx +++ b/web/src/app/admin/documents/sets/hooks.tsx @@ -1,5 +1,5 @@ import { errorHandlingFetcher } from "@/lib/fetcher"; -import { DocumentSet } from "@/lib/types"; +import { DocumentSetSummary } from "@/lib/types"; import useSWR, { mutate } from "swr"; const DOCUMENT_SETS_URL = "/api/manage/document-set"; @@ -13,7 +13,7 @@ export function refreshDocumentSets() { export function useDocumentSets(getEditable: boolean = false) { const url = getEditable ? GET_EDITABLE_DOCUMENT_SETS_URL : DOCUMENT_SETS_URL; - const swrResponse = useSWR(url, errorHandlingFetcher, { + const swrResponse = useSWR(url, errorHandlingFetcher, { refreshInterval: 5000, // 5 seconds }); diff --git a/web/src/app/admin/documents/sets/page.tsx b/web/src/app/admin/documents/sets/page.tsx index 1727e65da02..00251925413 100644 --- a/web/src/app/admin/documents/sets/page.tsx +++ b/web/src/app/admin/documents/sets/page.tsx @@ -14,6 +14,7 @@ import Text from "@/components/ui/text"; import Title from "@/components/ui/title"; import { Separator } from "@/components/ui/separator"; import { DocumentSet } from "@/lib/types"; +import { DocumentSetSummary } from "@/lib/types"; import { useState } from "react"; import { useDocumentSets } from "./hooks"; import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle"; @@ -105,7 +106,7 @@ const EditRow = ({ documentSet, isEditable, }: { - documentSet: DocumentSet; + documentSet: DocumentSetSummary; isEditable: boolean; }) => { const router = useRouter(); @@ -155,11 +156,11 @@ const EditRow = ({ }; interface DocumentFeedbackTableProps { - documentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; refresh: () => void; refreshEditable: () => void; setPopup: (popupSpec: PopupSpec | null) => void; - editableDocumentSets: DocumentSet[]; + editableDocumentSets: DocumentSetSummary[]; } const DocumentSetTable = ({ @@ -222,43 +223,47 @@ const DocumentSetTable = ({
{/* Regular Connectors */} - {documentSet.cc_pair_descriptors.map( - (ccPairDescriptor, ind) => { + {documentSet.cc_pair_summaries.map( + (ccPairSummary, ind) => { return (
- +
+ +
+ {ccPairSummary.name || "Unnamed"} +
+
); } )} {/* Federated Connectors */} - {documentSet.federated_connectors && - documentSet.federated_connectors.length > 0 && ( + {documentSet.federated_connector_summaries && + documentSet.federated_connector_summaries.length > + 0 && ( <> - {documentSet.cc_pair_descriptors.length > 0 && ( + {documentSet.cc_pair_summaries.length > 0 && (
)} - {documentSet.federated_connectors.map( + {documentSet.federated_connector_summaries.map( (federatedConnector, ind) => { return (
Up to Date - ) : documentSet.cc_pair_descriptors.length > 0 || - (documentSet.federated_connectors && - documentSet.federated_connectors.length > 0) ? ( + ) : documentSet.cc_pair_summaries.length > 0 || + (documentSet.federated_connector_summaries && + documentSet.federated_connector_summaries.length > + 0) ? ( Syncing diff --git a/web/src/components/documentSet/DocumentSetSelectable.tsx b/web/src/components/documentSet/DocumentSetSelectable.tsx index a289b06aab0..1b9f911578b 100644 --- a/web/src/components/documentSet/DocumentSetSelectable.tsx +++ b/web/src/components/documentSet/DocumentSetSelectable.tsx @@ -1,6 +1,6 @@ "use client"; -import { DocumentSet, ValidSources } from "@/lib/types"; +import { DocumentSetSummary, ValidSources } from "@/lib/types"; import { CustomCheckbox } from "../CustomCheckbox"; import { SourceIcon } from "../SourceIcon"; import { @@ -17,7 +17,7 @@ export function DocumentSetSelectable({ disabled, disabledTooltip, }: { - documentSet: DocumentSet; + documentSet: DocumentSetSummary; isSelected: boolean; onSelect: () => void; disabled?: boolean; @@ -25,8 +25,8 @@ export function DocumentSetSelectable({ }) { // Collect unique connector sources const uniqueSources = new Set(); - documentSet.cc_pair_descriptors.forEach((ccPairDescriptor) => { - uniqueSources.add(ccPairDescriptor.connector.source); + documentSet.cc_pair_summaries.forEach((ccPairSummary) => { + uniqueSources.add(ccPairSummary.source); }); return ( diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 4f94ab2122e..47ea3b97b3b 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -269,6 +269,33 @@ export interface FederatedConnectorDescriptor { entities: Record; } +// Simplified interfaces with minimal data +export interface CCPairSummary { + id: number; + name: string | null; + source: ValidSources; + access_type: AccessType; +} + +export interface FederatedConnectorSummary { + id: number; + name: string; + source: string; + entities: Record; +} + +export interface DocumentSetSummary { + id: number; + name: string; + description: string; + cc_pair_summaries: CCPairSummary[]; + is_up_to_date: boolean; + is_public: boolean; + users: string[]; + groups: number[]; + federated_connector_summaries: FederatedConnectorSummary[]; +} + export interface DocumentSet { id: number; name: string; From b41c43fffcb7f3df115f98434eaf26fdf73ae49e Mon Sep 17 00:00:00 2001 From: Weves Date: Sat, 12 Jul 2025 16:49:19 -0700 Subject: [PATCH 2/5] Fix type errors --- web/package-lock.json | 12 ++-- web/package.json | 4 +- .../app/admin/assistants/AssistantEditor.tsx | 4 +- web/src/app/admin/assistants/interfaces.ts | 4 +- .../SlackChannelConfigCreationForm.tsx | 4 +- .../channels/SlackChannelConfigFormFields.tsx | 58 +++++++------------ .../bots/[bot-id]/channels/[id]/page.tsx | 8 ++- .../admin/bots/[bot-id]/channels/new/page.tsx | 5 +- .../app/admin/documents/explorer/Explorer.tsx | 4 +- web/src/app/admin/documents/sets/page.tsx | 1 - web/src/app/chat/input/ChatInputBar.tsx | 4 +- web/src/components/context/ChatContext.tsx | 11 +++- web/src/components/context/SearchContext.tsx | 4 +- web/src/components/filters/SourceSelector.tsx | 4 +- web/src/components/search/SearchBar.tsx | 6 +- .../search/filtering/FilterPopup.tsx | 10 ++-- .../assistants/fetchPersonaEditorInfoSS.ts | 7 ++- web/src/lib/chat/fetchChatData.ts | 6 +- web/src/lib/chat/fetchSomeChatData.ts | 6 +- web/src/lib/filters.ts | 6 +- web/src/lib/search/utilsSS.ts | 6 +- web/src/lib/sources.ts | 6 +- web/src/lib/types.ts | 14 +---- 23 files changed, 88 insertions(+), 106 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index 196278153b7..59da1ffa5f9 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -84,7 +84,6 @@ "tailwind-merge": "^2.5.4", "tailwindcss": "^3.3.1", "tailwindcss-animate": "^1.0.7", - "typescript": "5.0.3", "uuid": "^9.0.1", "vaul": "^1.1.1", "yup": "^1.4.0" @@ -102,7 +101,8 @@ "jest": "^29.7.0", "prettier": "3.1.0", "ts-jest": "^29.2.5", - "ts-unused-exports": "^11.0.1" + "ts-unused-exports": "^11.0.1", + "typescript": "^5.8.3" } }, "node_modules/@alloc/quick-lru": { @@ -17708,14 +17708,16 @@ } }, "node_modules/typescript": { - "version": "5.0.3", - "license": "Apache-2.0", + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/unbox-primitive": { diff --git a/web/package.json b/web/package.json index 7fda098a821..47fab6b121b 100644 --- a/web/package.json +++ b/web/package.json @@ -90,7 +90,6 @@ "tailwind-merge": "^2.5.4", "tailwindcss": "^3.3.1", "tailwindcss-animate": "^1.0.7", - "typescript": "5.0.3", "uuid": "^9.0.1", "vaul": "^1.1.1", "yup": "^1.4.0" @@ -108,7 +107,8 @@ "jest": "^29.7.0", "prettier": "3.1.0", "ts-jest": "^29.2.5", - "ts-unused-exports": "^11.0.1" + "ts-unused-exports": "^11.0.1", + "typescript": "^5.8.3" }, "overrides": { "react-is": "^19.0.0-rc-69d4b800-20241021" diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx index 641406c3ab2..949abd66a5e 100644 --- a/web/src/app/admin/assistants/AssistantEditor.tsx +++ b/web/src/app/admin/assistants/AssistantEditor.tsx @@ -5,7 +5,7 @@ import { Option } from "@/components/Dropdown"; import { generateRandomIconShape } from "@/lib/assistantIconUtils"; import { CCPairBasicInfo, - DocumentSet, + DocumentSetSummary, User, UserGroup, UserRole, @@ -125,7 +125,7 @@ export function AssistantEditor({ }: { existingPersona?: FullPersona | null; ccPairs: CCPairBasicInfo[]; - documentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; user: User | null; defaultPublic: boolean; llmProviders: LLMProviderView[]; diff --git a/web/src/app/admin/assistants/interfaces.ts b/web/src/app/admin/assistants/interfaces.ts index fe88e3a9449..89666315d93 100644 --- a/web/src/app/admin/assistants/interfaces.ts +++ b/web/src/app/admin/assistants/interfaces.ts @@ -1,5 +1,5 @@ import { ToolSnapshot } from "@/lib/tools/interfaces"; -import { DocumentSet, MinimalUserSnapshot } from "@/lib/types"; +import { DocumentSetSummary, MinimalUserSnapshot } from "@/lib/types"; export interface StarterMessageBase { message: string; @@ -25,7 +25,7 @@ export interface MinimalPersonaSnapshot { description: string; tools: ToolSnapshot[]; starter_messages: StarterMessage[] | null; - document_sets: DocumentSet[]; + document_sets: DocumentSetSummary[]; llm_model_version_override?: string; llm_model_provider_override?: string; diff --git a/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm.tsx b/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm.tsx index 5b4c2520246..5eba1795cf8 100644 --- a/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm.tsx +++ b/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm.tsx @@ -5,7 +5,7 @@ import { Formik, Form } from "formik"; import * as Yup from "yup"; import { usePopup } from "@/components/admin/connectors/Popup"; import { - DocumentSet, + DocumentSetSummary, SlackChannelConfig, SlackBotResponseType, } from "@/lib/types"; @@ -29,7 +29,7 @@ export const SlackChannelConfigCreationForm = ({ existingSlackChannelConfig, }: { slack_bot_id: number; - documentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; personas: MinimalPersonaSnapshot[]; standardAnswerCategoryResponse: StandardAnswerCategoryResponse; existingSlackChannelConfig?: SlackChannelConfig; diff --git a/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx b/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx index ce0d62807f5..119a9a85941 100644 --- a/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx +++ b/web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx @@ -2,7 +2,7 @@ import React, { useState, useEffect, useMemo } from "react"; import { FieldArray, useFormikContext, ErrorMessage, Field } from "formik"; -import { CCPairDescriptor, DocumentSet } from "@/lib/types"; +import { CCPairDescriptor, DocumentSetSummary } from "@/lib/types"; import { Label, SelectorFormField, @@ -39,7 +39,6 @@ import { import { Separator } from "@/components/ui/separator"; import { CheckFormField } from "@/components/ui/CheckField"; -import { DocumentSetSummary } from "@/lib/types"; export interface SlackChannelConfigFormFieldsProps { isUpdate: boolean; @@ -74,37 +73,24 @@ export function SlackChannelConfigFormFields({ useState(false); // Helper function to check if a document set contains sync connectors - const documentSetContainsSync = ( - documentSet: DocumentSetSummary | DocumentSet - ) => { - // Check if it's a DocumentSetSummary (has cc_pair_summaries) - if ("cc_pair_summaries" in documentSet) { - return documentSet.cc_pair_summaries.some( - (summary) => summary.access_type === "sync" - ); - } - // Otherwise it's a DocumentSet (has cc_pair_descriptors) - return documentSet.cc_pair_descriptors.some( - (descriptor) => descriptor.access_type === "sync" + const documentSetContainsSync = (documentSet: DocumentSetSummary) => { + return documentSet.cc_pair_summaries.some( + (summary) => summary.access_type === "sync" ); }; // Helper function to check if a document set contains private connectors - const documentSetContainsPrivate = ( - documentSet: DocumentSetSummary | DocumentSet - ) => { - // Check if it's a DocumentSetSummary (has cc_pair_summaries) - if ("cc_pair_summaries" in documentSet) { - return documentSet.cc_pair_summaries.some( - (summary) => summary.access_type === "private" - ); - } - // Otherwise it's a DocumentSet (has cc_pair_descriptors) - return documentSet.cc_pair_descriptors.some( - (descriptor) => descriptor.access_type === "private" + const documentSetContainsPrivate = (documentSet: DocumentSetSummary) => { + return documentSet.cc_pair_summaries.some( + (summary) => summary.access_type === "private" ); }; + // Helper function to get cc_pair_summaries from DocumentSetSummary + const getCcPairSummaries = (documentSet: DocumentSetSummary) => { + return documentSet.cc_pair_summaries; + }; + const [syncEnabledAssistants, availableAssistants] = useMemo(() => { const sync: MinimalPersonaSnapshot[] = []; const available: MinimalPersonaSnapshot[] = []; @@ -122,15 +108,14 @@ export function SlackChannelConfigFormFields({ }, [searchEnabledAssistants]); const unselectableSets = useMemo(() => { - return documentSets.filter((ds) => - ds.cc_pair_summaries.some((summary) => summary.access_type === "sync") - ); + return documentSets.filter(documentSetContainsSync); }, [documentSets]); const memoizedPrivateConnectors = useMemo(() => { const uniqueDescriptors = new Map(); - documentSets.forEach((ds) => { - ds.cc_pair_summaries.forEach((summary) => { + documentSets.forEach((ds: DocumentSetSummary) => { + const ccPairSummaries = getCcPairSummaries(ds); + ccPairSummaries.forEach((summary: any) => { if ( summary.access_type === "private" && !uniqueDescriptors.has(summary.id) @@ -142,6 +127,10 @@ export function SlackChannelConfigFormFields({ return Array.from(uniqueDescriptors.values()); }, [documentSets]); + const selectableSets = useMemo(() => { + return documentSets.filter((ds) => !documentSetContainsSync(ds)); + }, [documentSets]); + useEffect(() => { const invalidSelected = values.document_sets.filter((dsId: number) => unselectableSets.some((us) => us.id === dsId) @@ -179,13 +168,6 @@ export function SlackChannelConfigFormFields({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [values.knowledge_source, values.document_sets, values.persona_id]); - const selectableSets = useMemo(() => { - return documentSets.filter( - (ds) => - !ds.cc_pair_summaries.some((summary) => summary.access_type === "sync") - ); - }, [documentSets]); - return ( <>
diff --git a/web/src/app/admin/bots/[bot-id]/channels/[id]/page.tsx b/web/src/app/admin/bots/[bot-id]/channels/[id]/page.tsx index 383312bdf90..4027ea2c450 100644 --- a/web/src/app/admin/bots/[bot-id]/channels/[id]/page.tsx +++ b/web/src/app/admin/bots/[bot-id]/channels/[id]/page.tsx @@ -3,7 +3,11 @@ import { SourceIcon } from "@/components/SourceIcon"; import { SlackChannelConfigCreationForm } from "../SlackChannelConfigCreationForm"; import { fetchSS } from "@/lib/utilsSS"; import { ErrorCallout } from "@/components/ErrorCallout"; -import { DocumentSet, SlackChannelConfig, ValidSources } from "@/lib/types"; +import { + DocumentSetSummary, + SlackChannelConfig, + ValidSources, +} from "@/lib/types"; import { BackButton } from "@/components/BackButton"; import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh"; import { @@ -68,7 +72,7 @@ async function EditslackChannelConfigPage(props: { ); } const response = await documentSetsResponse.json(); - const documentSets = response as DocumentSet[]; + const documentSets = response as DocumentSetSummary[]; if (assistantsFetchError) { return ( diff --git a/web/src/app/admin/bots/[bot-id]/channels/new/page.tsx b/web/src/app/admin/bots/[bot-id]/channels/new/page.tsx index 56f4848c7b9..f73fea3a892 100644 --- a/web/src/app/admin/bots/[bot-id]/channels/new/page.tsx +++ b/web/src/app/admin/bots/[bot-id]/channels/new/page.tsx @@ -2,7 +2,7 @@ import { AdminPageTitle } from "@/components/admin/Title"; import { SlackChannelConfigCreationForm } from "../SlackChannelConfigCreationForm"; import { fetchSS } from "@/lib/utilsSS"; import { ErrorCallout } from "@/components/ErrorCallout"; -import { DocumentSet, ValidSources } from "@/lib/types"; +import { DocumentSetSummary, ValidSources } from "@/lib/types"; import { BackButton } from "@/components/BackButton"; import { fetchAssistantsSS } from "@/lib/assistants/fetchAssistantsSS"; import { getStandardAnswerCategoriesIfEE } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE"; @@ -40,7 +40,8 @@ async function NewChannelConfigPage(props: { /> ); } - const documentSets = (await documentSetsResponse.json()) as DocumentSet[]; + const documentSets = + (await documentSetsResponse.json()) as DocumentSetSummary[]; if (assistantsResponse[1]) { return ( diff --git a/web/src/app/admin/documents/explorer/Explorer.tsx b/web/src/app/admin/documents/explorer/Explorer.tsx index 0fc08947295..2b9819ea6a9 100644 --- a/web/src/app/admin/documents/explorer/Explorer.tsx +++ b/web/src/app/admin/documents/explorer/Explorer.tsx @@ -14,7 +14,7 @@ import { useRouter } from "next/navigation"; import { useFilters } from "@/lib/hooks"; import { buildFilters } from "@/lib/search/utils"; import { DocumentUpdatedAtBadge } from "@/components/search/DocumentUpdatedAtBadge"; -import { DocumentSet } from "@/lib/types"; +import { DocumentSetSummary } from "@/lib/types"; import { SourceIcon } from "@/components/SourceIcon"; import { Connector } from "@/lib/connectors/connectors"; import { HorizontalFilters } from "@/components/filters/SourceSelector"; @@ -110,7 +110,7 @@ export function Explorer({ }: { initialSearchValue: string | undefined; connectors: Connector[]; - documentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; }) { const router = useRouter(); const { popup, setPopup } = usePopup(); diff --git a/web/src/app/admin/documents/sets/page.tsx b/web/src/app/admin/documents/sets/page.tsx index 00251925413..0fc2e8e0187 100644 --- a/web/src/app/admin/documents/sets/page.tsx +++ b/web/src/app/admin/documents/sets/page.tsx @@ -13,7 +13,6 @@ import { import Text from "@/components/ui/text"; import Title from "@/components/ui/title"; import { Separator } from "@/components/ui/separator"; -import { DocumentSet } from "@/lib/types"; import { DocumentSetSummary } from "@/lib/types"; import { useState } from "react"; import { useDocumentSets } from "./hooks"; diff --git a/web/src/app/chat/input/ChatInputBar.tsx b/web/src/app/chat/input/ChatInputBar.tsx index f0cdd63a3d5..5594c4b04fa 100644 --- a/web/src/app/chat/input/ChatInputBar.tsx +++ b/web/src/app/chat/input/ChatInputBar.tsx @@ -29,7 +29,7 @@ import { UnconfiguredLlmProviderText } from "@/components/chat/UnconfiguredLlmPr import { useAssistants } from "@/components/context/AssistantsContext"; import { CalendarIcon, TagIcon, XIcon, FolderIcon } from "lucide-react"; import { FilterPopup } from "@/components/search/filtering/FilterPopup"; -import { DocumentSet, Tag } from "@/lib/types"; +import { DocumentSetSummary, Tag } from "@/lib/types"; import { SourceIcon } from "@/components/SourceIcon"; import { getFormattedDateRangeString } from "@/lib/dateUtils"; import { truncateString } from "@/lib/utils"; @@ -193,7 +193,7 @@ interface ChatInputBarProps { textAreaRef: React.RefObject; filterManager: FilterManager; availableSources: SourceMetadata[]; - availableDocumentSets: DocumentSet[]; + availableDocumentSets: DocumentSetSummary[]; availableTags: Tag[]; retrievalEnabled: boolean; proSearchEnabled: boolean; diff --git a/web/src/components/context/ChatContext.tsx b/web/src/components/context/ChatContext.tsx index 092559c195b..4ee2820531a 100644 --- a/web/src/components/context/ChatContext.tsx +++ b/web/src/components/context/ChatContext.tsx @@ -1,7 +1,12 @@ "use client"; import React, { createContext, useContext, useState } from "react"; -import { CCPairBasicInfo, DocumentSet, Tag, ValidSources } from "@/lib/types"; +import { + CCPairBasicInfo, + DocumentSetSummary, + Tag, + ValidSources, +} from "@/lib/types"; import { ChatSession, InputPrompt } from "@/app/chat/interfaces"; import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces"; import { Folder } from "@/app/chat/folders/interfaces"; @@ -14,8 +19,8 @@ interface ChatContextProps { availableSources: ValidSources[]; ccPairs: CCPairBasicInfo[]; tags: Tag[]; - documentSets: DocumentSet[]; - availableDocumentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; + availableDocumentSets: DocumentSetSummary[]; availableTags: Tag[]; llmProviders: LLMProviderDescriptor[]; folders: Folder[]; diff --git a/web/src/components/context/SearchContext.tsx b/web/src/components/context/SearchContext.tsx index 3150747b483..b1a40491cef 100644 --- a/web/src/components/context/SearchContext.tsx +++ b/web/src/components/context/SearchContext.tsx @@ -1,7 +1,7 @@ "use client"; import React, { createContext, useContext } from "react"; -import { CCPairBasicInfo, DocumentSet, Tag } from "@/lib/types"; +import { CCPairBasicInfo, DocumentSetSummary, Tag } from "@/lib/types"; import { Persona } from "@/app/admin/assistants/interfaces"; import { ChatSession } from "@/app/chat/interfaces"; @@ -9,7 +9,7 @@ interface SearchContextProps { querySessions: ChatSession[]; ccPairs: CCPairBasicInfo[]; tags: Tag[]; - documentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; assistants: Persona[]; agenticSearchEnabled: boolean; disabledAgentic: boolean; diff --git a/web/src/components/filters/SourceSelector.tsx b/web/src/components/filters/SourceSelector.tsx index a5d1c4ccea1..aa9cd16324a 100644 --- a/web/src/components/filters/SourceSelector.tsx +++ b/web/src/components/filters/SourceSelector.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { DocumentSet, Tag, ValidSources } from "@/lib/types"; +import { DocumentSetSummary, Tag, ValidSources } from "@/lib/types"; import { SourceMetadata } from "@/lib/search/interfaces"; import { InfoIcon, defaultTailwindCSS } from "@/components/icons/icons"; import { HoverPopup } from "@/components/HoverPopup"; @@ -34,7 +34,7 @@ export interface SourceSelectorProps { setSelectedDocumentSets: React.Dispatch>; selectedTags: Tag[]; setSelectedTags: React.Dispatch>; - availableDocumentSets: DocumentSet[]; + availableDocumentSets: DocumentSetSummary[]; existingSources: ValidSources[]; availableTags: Tag[]; toggleFilters: () => void; diff --git a/web/src/components/search/SearchBar.tsx b/web/src/components/search/SearchBar.tsx index 93258935d60..db62b714832 100644 --- a/web/src/components/search/SearchBar.tsx +++ b/web/src/components/search/SearchBar.tsx @@ -9,9 +9,9 @@ interface FullSearchBarProps { agentic?: boolean; toggleAgentic?: () => void; ccPairs: CCPairBasicInfo[]; - documentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; filterManager: any; // You might want to replace 'any' with a more specific type - finalAvailableDocumentSets: DocumentSet[]; + finalAvailableDocumentSets: DocumentSetSummary[]; finalAvailableSources: string[]; tags: Tag[]; showingSidebar: boolean; @@ -27,7 +27,7 @@ import { useRef } from "react"; import { SendIcon } from "../icons/icons"; import { Separator } from "@/components/ui/separator"; import KeyboardSymbol from "@/lib/browserUtilities"; -import { CCPairBasicInfo, DocumentSet, Tag } from "@/lib/types"; +import { CCPairBasicInfo, DocumentSetSummary, Tag } from "@/lib/types"; import { HorizontalSourceSelector } from "./filtering/HorizontalSourceSelector"; export const AnimatedToggle = ({ diff --git a/web/src/components/search/filtering/FilterPopup.tsx b/web/src/components/search/filtering/FilterPopup.tsx index 514e1a0794e..cfe9ad842c9 100644 --- a/web/src/components/search/filtering/FilterPopup.tsx +++ b/web/src/components/search/filtering/FilterPopup.tsx @@ -14,7 +14,7 @@ import { FiChevronDown, } from "react-icons/fi"; import { FilterManager } from "@/lib/hooks"; -import { DocumentSet, Tag } from "@/lib/types"; +import { DocumentSetSummary, Tag } from "@/lib/types"; import { SourceMetadata } from "@/lib/search/interfaces"; import { Checkbox } from "@/components/ui/checkbox"; import { Separator } from "@/components/ui/separator"; @@ -37,7 +37,7 @@ interface FilterPopupProps { tooltipContent?: React.ReactNode; }; availableSources: SourceMetadata[]; - availableDocumentSets: DocumentSet[]; + availableDocumentSets: DocumentSetSummary[]; availableTags: Tag[]; } @@ -61,7 +61,7 @@ export function FilterPopup({ const [currentDate, setCurrentDate] = useState(new Date()); const [documentSetSearch, setDocumentSetSearch] = useState(""); const [filteredDocumentSets, setFilteredDocumentSets] = useState< - DocumentSet[] + DocumentSetSummary[] >(availableDocumentSets); useEffect(() => { @@ -249,10 +249,10 @@ export function FilterPopup({ } }; - const isDocumentSetSelected = (docSet: DocumentSet) => + const isDocumentSetSelected = (docSet: DocumentSetSummary) => filterManager.selectedDocumentSets.includes(docSet.name); - const toggleDocumentSet = (docSet: DocumentSet) => { + const toggleDocumentSet = (docSet: DocumentSetSummary) => { filterManager.setSelectedDocumentSets((prev) => prev.includes(docSet.name) ? prev.filter((id) => id !== docSet.name) diff --git a/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts b/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts index 8b803a1d9e7..e1e12735966 100644 --- a/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts +++ b/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts @@ -1,5 +1,5 @@ import { FullPersona } from "@/app/admin/assistants/interfaces"; -import { CCPairBasicInfo, DocumentSet, User } from "../types"; +import { CCPairBasicInfo, DocumentSetSummary, User } from "../types"; import { getCurrentUserSS } from "../userSS"; import { fetchSS } from "../utilsSS"; import { LLMProviderView } from "@/app/admin/configuration/llm/interfaces"; @@ -12,7 +12,7 @@ export async function fetchAssistantEditorInfoSS( | [ { ccPairs: CCPairBasicInfo[]; - documentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; llmProviders: LLMProviderView[]; user: User | null; existingPersona: FullPersona | null; @@ -67,7 +67,8 @@ export async function fetchAssistantEditorInfoSS( `Failed to fetch document sets - ${await documentSetsResponse.text()}`, ]; } - const documentSets = (await documentSetsResponse.json()) as DocumentSet[]; + const documentSets = + (await documentSetsResponse.json()) as DocumentSetSummary[]; if (!toolsResponse) { return [null, `Failed to fetch tools`]; diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts index 3bd013869b2..f3231e9189b 100644 --- a/web/src/lib/chat/fetchChatData.ts +++ b/web/src/lib/chat/fetchChatData.ts @@ -6,7 +6,7 @@ import { import { fetchSS } from "@/lib/utilsSS"; import { CCPairBasicInfo, - DocumentSet, + DocumentSetSummary, Tag, User, ValidSources, @@ -35,7 +35,7 @@ interface FetchChatDataResult { chatSessions: ChatSession[]; ccPairs: CCPairBasicInfo[]; availableSources: ValidSources[]; - documentSets: DocumentSet[]; + documentSets: DocumentSetSummary[]; tags: Tag[]; llmProviders: LLMProviderDescriptor[]; folders: Folder[]; @@ -167,7 +167,7 @@ export async function fetchChatData(searchParams: { new Date(b.time_updated).getTime() - new Date(a.time_updated).getTime() ); - let documentSets: DocumentSet[] = []; + let documentSets: DocumentSetSummary[] = []; if (documentSetsResponse?.ok) { documentSets = await documentSetsResponse.json(); } else { diff --git a/web/src/lib/chat/fetchSomeChatData.ts b/web/src/lib/chat/fetchSomeChatData.ts index d895ae79b6a..14b43d7b689 100644 --- a/web/src/lib/chat/fetchSomeChatData.ts +++ b/web/src/lib/chat/fetchSomeChatData.ts @@ -6,7 +6,7 @@ import { import { fetchSS } from "@/lib/utilsSS"; import { CCPairBasicInfo, - DocumentSet, + DocumentSetSummary, Tag, User, ValidSources, @@ -31,7 +31,7 @@ interface FetchChatDataResult { chatSessions?: ChatSession[]; ccPairs?: CCPairBasicInfo[]; availableSources?: ValidSources[]; - documentSets?: DocumentSet[]; + documentSets?: DocumentSetSummary[]; assistants?: Persona[]; tags?: Tag[]; llmProviders?: LLMProviderDescriptor[]; @@ -123,7 +123,7 @@ export async function fetchSomeChatData( break; case "documentSets": result.documentSets = result?.ok - ? ((await result.json()) as DocumentSet[]) + ? ((await result.json()) as DocumentSetSummary[]) : []; break; case "assistants": diff --git a/web/src/lib/filters.ts b/web/src/lib/filters.ts index 078398e64b1..e11267f92cc 100644 --- a/web/src/lib/filters.ts +++ b/web/src/lib/filters.ts @@ -1,5 +1,5 @@ import { Persona } from "@/app/admin/assistants/interfaces"; -import { DocumentSet, ValidSources } from "./types"; +import { DocumentSetSummary, ValidSources } from "./types"; import { getSourcesForPersona } from "./sources"; export function computeAvailableFilters({ @@ -9,8 +9,8 @@ export function computeAvailableFilters({ }: { selectedPersona: Persona | undefined | null; availableSources: ValidSources[]; - availableDocumentSets: DocumentSet[]; -}): [ValidSources[], DocumentSet[]] { + availableDocumentSets: DocumentSetSummary[]; +}): [ValidSources[], DocumentSetSummary[]] { const finalAvailableSources = selectedPersona && selectedPersona.document_sets.length ? getSourcesForPersona(selectedPersona) diff --git a/web/src/lib/search/utilsSS.ts b/web/src/lib/search/utilsSS.ts index dfee8c361cc..80c76751fbd 100644 --- a/web/src/lib/search/utilsSS.ts +++ b/web/src/lib/search/utilsSS.ts @@ -1,4 +1,4 @@ -import { DocumentSet } from "../types"; +import { DocumentSetSummary } from "../types"; import { fetchSS } from "../utilsSS"; import { Connector } from "../connectors/connectors"; @@ -17,9 +17,9 @@ export async function fetchValidFilterInfo() { ); } - let documentSets = [] as DocumentSet[]; + let documentSets = [] as DocumentSetSummary[]; if (documentSetResponse.ok) { - documentSets = (await documentSetResponse.json()) as DocumentSet[]; + documentSets = (await documentSetResponse.json()) as DocumentSetSummary[]; } else { console.log( `Failed to fetch document sets - ${documentSetResponse.status} - ${documentSetResponse.statusText}` diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts index ce0bd90da64..48a5c44ddaf 100644 --- a/web/src/lib/sources.ts +++ b/web/src/lib/sources.ts @@ -418,9 +418,9 @@ export function getSourceMetadataForSources(sources: ValidSources[]) { export function getSourcesForPersona(persona: Persona): ValidSources[] { const personaSources: ValidSources[] = []; persona.document_sets.forEach((documentSet) => { - documentSet.cc_pair_descriptors.forEach((ccPair) => { - if (!personaSources.includes(ccPair.connector.source)) { - personaSources.push(ccPair.connector.source); + documentSet.cc_pair_summaries.forEach((ccPair) => { + if (!personaSources.includes(ccPair.source)) { + personaSources.push(ccPair.source); } }); }); diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 47ea3b97b3b..70912588982 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -296,18 +296,6 @@ export interface DocumentSetSummary { federated_connector_summaries: FederatedConnectorSummary[]; } -export interface DocumentSet { - id: number; - name: string; - description: string; - cc_pair_descriptors: CCPairDescriptor[]; - is_up_to_date: boolean; - is_public: boolean; - users: string[]; - groups: number[]; - federated_connectors: FederatedConnectorDescriptor[]; -} - export interface Tag { tag_key: string; tag_value: string; @@ -393,7 +381,7 @@ export interface UserGroup { users: User[]; curator_ids: string[]; cc_pairs: CCPairDescriptor[]; - document_sets: DocumentSet[]; + document_sets: DocumentSetSummary[]; personas: Persona[]; is_up_to_date: boolean; is_up_for_deletion: boolean; From 31214cf7c3e51d13eae099c85974be76a4568367 Mon Sep 17 00:00:00 2001 From: Weves Date: Sun, 13 Jul 2025 13:52:34 -0700 Subject: [PATCH 3/5] Fix tests --- .../common_utils/managers/document_set.py | 19 +++++++++++++------ .../tests/integration/common_utils/reset.py | 5 +++++ .../integration/common_utils/test_models.py | 1 + 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/backend/tests/integration/common_utils/managers/document_set.py b/backend/tests/integration/common_utils/managers/document_set.py index f9b73cb2971..ca537d3f534 100644 --- a/backend/tests/integration/common_utils/managers/document_set.py +++ b/backend/tests/integration/common_utils/managers/document_set.py @@ -1,4 +1,6 @@ import time +from typing import Any +from uuid import UUID from uuid import uuid4 import requests @@ -19,6 +21,7 @@ def create( is_public: bool = True, users: list[str] | None = None, groups: list[int] | None = None, + federated_connectors: list[dict[str, Any]] | None = None, user_performing_action: DATestUser | None = None, ) -> DATestDocumentSet: if name is None: @@ -29,8 +32,9 @@ def create( "description": description or name, "cc_pair_ids": cc_pair_ids or [], "is_public": is_public, - "users": users or [], + "users": [str(UUID(user_id)) for user_id in (users or [])], "groups": groups or [], + "federated_connectors": federated_connectors or [], } response = requests.post( @@ -53,6 +57,7 @@ def create( is_up_to_date=True, users=users or [], groups=groups or [], + federated_connectors=federated_connectors or [], ) @staticmethod @@ -65,8 +70,9 @@ def edit( "description": document_set.description, "cc_pair_ids": document_set.cc_pair_ids, "is_public": document_set.is_public, - "users": document_set.users, + "users": [str(UUID(user_id)) for user_id in document_set.users], "groups": document_set.groups, + "federated_connectors": document_set.federated_connectors, } response = requests.patch( f"{API_SERVER_URL}/manage/admin/document-set", @@ -114,13 +120,12 @@ def get_all( id=doc_set["id"], name=doc_set["name"], description=doc_set["description"], - cc_pair_ids=[ - cc_pair["id"] for cc_pair in doc_set["cc_pair_descriptors"] - ], + cc_pair_ids=[cc_pair["id"] for cc_pair in doc_set["cc_pair_summaries"]], is_public=doc_set["is_public"], is_up_to_date=doc_set["is_up_to_date"], - users=doc_set["users"], + users=[str(user_id) for user_id in doc_set["users"]], groups=doc_set["groups"], + federated_connectors=doc_set["federated_connector_summaries"], ) for doc_set in response.json() ] @@ -186,6 +191,8 @@ def verify( and doc_set.is_public == document_set.is_public and set(doc_set.users) == set(document_set.users) and set(doc_set.groups) == set(document_set.groups) + and doc_set.federated_connectors + == document_set.federated_connectors ): return if not verify_deleted: diff --git a/backend/tests/integration/common_utils/reset.py b/backend/tests/integration/common_utils/reset.py index 8f25e19a3e7..9e167759c6f 100644 --- a/backend/tests/integration/common_utils/reset.py +++ b/backend/tests/integration/common_utils/reset.py @@ -1,4 +1,5 @@ import logging +import os import time from types import SimpleNamespace @@ -398,6 +399,10 @@ def reset_vespa_multitenant() -> None: def reset_all() -> None: + if os.environ.get("SKIP_RESET", "").lower() == "true": + logger.info("Skipping reset.") + return + logger.info("Resetting Postgres...") reset_postgres() logger.info("Resetting Vespa...") diff --git a/backend/tests/integration/common_utils/test_models.py b/backend/tests/integration/common_utils/test_models.py index 4d5be715d5c..c6e641516d5 100644 --- a/backend/tests/integration/common_utils/test_models.py +++ b/backend/tests/integration/common_utils/test_models.py @@ -117,6 +117,7 @@ class DATestDocumentSet(BaseModel): is_up_to_date: bool users: list[str] = Field(default_factory=list) groups: list[int] = Field(default_factory=list) + federated_connectors: list[dict[str, Any]] = Field(default_factory=list) class DATestPersona(BaseModel): From 79f98a6161671163384d74acf459185d2b8719b9 Mon Sep 17 00:00:00 2001 From: Weves Date: Sun, 13 Jul 2025 13:59:48 -0700 Subject: [PATCH 4/5] Fixes --- .../onyx/server/features/document_set/api.py | 6 +- .../server/features/document_set/models.py | 66 ++++++++++++------- backend/onyx/server/federated/models.py | 2 +- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/backend/onyx/server/features/document_set/api.py b/backend/onyx/server/features/document_set/api.py index 34165957137..8fcd65e3fec 100644 --- a/backend/onyx/server/features/document_set/api.py +++ b/backend/onyx/server/features/document_set/api.py @@ -19,7 +19,6 @@ from onyx.db.models import User from onyx.server.features.document_set.models import CheckDocSetPublicRequest from onyx.server.features.document_set.models import CheckDocSetPublicResponse -from onyx.server.features.document_set.models import DocumentSet from onyx.server.features.document_set.models import DocumentSetCreationRequest from onyx.server.features.document_set.models import DocumentSetSummary from onyx.server.features.document_set.models import DocumentSetUpdateRequest @@ -159,10 +158,7 @@ def list_document_sets_for_user( document_sets = fetch_all_document_sets_for_user( db_session=db_session, user=user, get_editable=get_editable ) - return [ - DocumentSetSummary.from_document_set(DocumentSet.from_model(ds)) - for ds in document_sets - ] + return [DocumentSetSummary.from_document_set(ds) for ds in document_sets] @router.get("/document-set-public") diff --git a/backend/onyx/server/features/document_set/models.py b/backend/onyx/server/features/document_set/models.py index 410bf0fc091..ba60e6f9b2a 100644 --- a/backend/onyx/server/features/document_set/models.py +++ b/backend/onyx/server/features/document_set/models.py @@ -5,6 +5,7 @@ from pydantic import Field from onyx.db.models import DocumentSet as DocumentSetDBModel +from onyx.db.models import FederatedConnector__DocumentSet from onyx.server.documents.models import CCPairSummary from onyx.server.documents.models import ConnectorCredentialPairDescriptor from onyx.server.documents.models import ConnectorSnapshot @@ -27,6 +28,26 @@ class FederatedConnectorDescriptor(BaseModel): source: str entities: dict[str, Any] + @classmethod + def from_federated_connector_mapping( + cls, fc_mapping: "FederatedConnector__DocumentSet" + ) -> "FederatedConnectorDescriptor": + """Create a descriptor from a federated connector mapping""" + return cls( + id=fc_mapping.federated_connector_id, + name=( + f"{fc_mapping.federated_connector.source.replace('_', ' ').title()}" + if fc_mapping.federated_connector + else "Unknown" + ), + source=( + fc_mapping.federated_connector.source + if fc_mapping.federated_connector + else "unknown" + ), + entities=fc_mapping.entities, + ) + class DocumentSetSummary(BaseModel): """Simplified document set model with minimal data for list views""" @@ -44,28 +65,36 @@ class DocumentSetSummary(BaseModel): ) @classmethod - def from_document_set(cls, document_set: "DocumentSet") -> "DocumentSetSummary": - """Create a summary from a full DocumentSet model""" + def from_document_set( + cls, document_set: DocumentSetDBModel + ) -> "DocumentSetSummary": + """Create a summary from a DocumentSet database model""" return cls( id=document_set.id, name=document_set.name, description=document_set.description, cc_pair_summaries=[ - CCPairSummary.from_cc_pair_descriptor(desc) - for desc in document_set.cc_pair_descriptors + CCPairSummary( + id=cc_pair.id, + name=cc_pair.name, + source=cc_pair.connector.source, + access_type=cc_pair.access_type, + ) + for cc_pair in document_set.connector_credential_pairs ], is_up_to_date=document_set.is_up_to_date, is_public=document_set.is_public, - users=document_set.users, - groups=document_set.groups, + users=[user.id for user in document_set.users], + groups=[group.id for group in document_set.groups], federated_connector_summaries=[ FederatedConnectorSummary( - id=fc.id, - name=fc.name, - source=fc.source, - entities=fc.entities, + id=fc_mapping.federated_connector_id, + name=f"{fc_mapping.federated_connector.source.replace('_', ' ').title()}", + source=fc_mapping.federated_connector.source, + entities=fc_mapping.entities, ) - for fc in document_set.federated_connectors + for fc_mapping in document_set.federated_connectors + if fc_mapping.federated_connector is not None ], ) @@ -146,19 +175,8 @@ def from_model(cls, document_set_model: DocumentSetDBModel) -> "DocumentSet": users=[user.id for user in document_set_model.users], groups=[group.id for group in document_set_model.groups], federated_connectors=[ - FederatedConnectorDescriptor( - id=fc_mapping.federated_connector_id, - name=( - f"{fc_mapping.federated_connector.source.replace('_', ' ').title()}" - if fc_mapping.federated_connector - else "Unknown" - ), - source=( - fc_mapping.federated_connector.source - if fc_mapping.federated_connector - else "unknown" - ), - entities=fc_mapping.entities, + FederatedConnectorDescriptor.from_federated_connector_mapping( + fc_mapping ) for fc_mapping in document_set_model.federated_connectors ], diff --git a/backend/onyx/server/federated/models.py b/backend/onyx/server/federated/models.py index 95b0748a1c5..471f536fb21 100644 --- a/backend/onyx/server/federated/models.py +++ b/backend/onyx/server/federated/models.py @@ -70,7 +70,7 @@ class FederatedConnectorSummary(BaseModel): id: int name: str - source: str # Using str to match FederatedConnectorDescriptor pattern + source: FederatedConnectorSource entities: dict[str, Any] @classmethod From f511616cae68cc9340d99523054d73449611950a Mon Sep 17 00:00:00 2001 From: Weves Date: Sun, 13 Jul 2025 14:00:49 -0700 Subject: [PATCH 5/5] Don't change packages --- web/package-lock.json | 12 +++++------- web/package.json | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index 59da1ffa5f9..196278153b7 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -84,6 +84,7 @@ "tailwind-merge": "^2.5.4", "tailwindcss": "^3.3.1", "tailwindcss-animate": "^1.0.7", + "typescript": "5.0.3", "uuid": "^9.0.1", "vaul": "^1.1.1", "yup": "^1.4.0" @@ -101,8 +102,7 @@ "jest": "^29.7.0", "prettier": "3.1.0", "ts-jest": "^29.2.5", - "ts-unused-exports": "^11.0.1", - "typescript": "^5.8.3" + "ts-unused-exports": "^11.0.1" } }, "node_modules/@alloc/quick-lru": { @@ -17708,16 +17708,14 @@ } }, "node_modules/typescript": { - "version": "5.8.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", - "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", - "dev": true, + "version": "5.0.3", + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=14.17" + "node": ">=12.20" } }, "node_modules/unbox-primitive": { diff --git a/web/package.json b/web/package.json index 47fab6b121b..7fda098a821 100644 --- a/web/package.json +++ b/web/package.json @@ -90,6 +90,7 @@ "tailwind-merge": "^2.5.4", "tailwindcss": "^3.3.1", "tailwindcss-animate": "^1.0.7", + "typescript": "5.0.3", "uuid": "^9.0.1", "vaul": "^1.1.1", "yup": "^1.4.0" @@ -107,8 +108,7 @@ "jest": "^29.7.0", "prettier": "3.1.0", "ts-jest": "^29.2.5", - "ts-unused-exports": "^11.0.1", - "typescript": "^5.8.3" + "ts-unused-exports": "^11.0.1" }, "overrides": { "react-is": "^19.0.0-rc-69d4b800-20241021"