SSR for projects

Author: Weves

.vscode/launch.template.jsonc

@@ -72,17 +72,17 @@ def run_jobs() -> None:
         "--queues=docprocessing",
     ]
 
-    cmd_worker_user_files_indexing = [
+    cmd_worker_user_files = [
         "celery",
         "-A",
-        "onyx.background.celery.versioned_apps.docfetching",
+        "onyx.background.celery.versioned_apps.user_file_processing",
         "worker",
         "--pool=threads",
         "--concurrency=1",
         "--prefetch-multiplier=1",
         "--loglevel=INFO",
-        "--hostname=user_files_indexing@%n",
-        "--queues=user_files_indexing",
+        "--hostname=user_file_processing@%n",
+        "--queues=user_file_processing,user_file_project_sync",
     ]
 
     cmd_worker_monitoring = [
@@ -152,8 +152,8 @@ def run_jobs() -> None:
         text=True,
     )
 
-    worker_user_files_indexing_process = subprocess.Popen(
-        cmd_worker_user_files_indexing,
+    worker_user_file_process = subprocess.Popen(
+        cmd_worker_user_files,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         text=True,
@@ -197,9 +197,9 @@ def run_jobs() -> None:
     worker_docprocessing_thread = threading.Thread(
         target=monitor_process, args=("DOCPROCESSING", worker_docprocessing_process)
     )
-    worker_user_files_indexing_thread = threading.Thread(
+    worker_user_file_thread = threading.Thread(
         target=monitor_process,
-        args=("USER_FILES_INDEX", worker_user_files_indexing_process),
+        args=("USER_FILE_PROCESSING", worker_user_file_process),
     )
     worker_monitoring_thread = threading.Thread(
         target=monitor_process, args=("MONITORING", worker_monitoring_process)
@@ -216,7 +216,7 @@ def run_jobs() -> None:
     worker_light_thread.start()
     worker_heavy_thread.start()
     worker_docprocessing_thread.start()
-    worker_user_files_indexing_thread.start()
+    worker_user_file_thread.start()
     worker_monitoring_thread.start()
     worker_kg_processing_thread.start()
     worker_docfetching_thread.start()
@@ -226,7 +226,7 @@ def run_jobs() -> None:
     worker_light_thread.join()
     worker_heavy_thread.join()
     worker_docprocessing_thread.join()
-    worker_user_files_indexing_thread.join()
+    worker_user_file_thread.join()
     worker_monitoring_thread.join()
     worker_kg_processing_thread.join()
     worker_docfetching_thread.join()

backend/scripts/dev_run_background_jobs.py

@@ -2,6 +2,7 @@
 
 import { useChatContext } from "@/components/context/ChatContext";
 import { ChatPage } from "./components/ChatPage";
+import { ProjectsProvider } from "./projects/ProjectsContext";
 import { useCallback, useState } from "react";
 
 export default function ChatLayout({
@@ -13,7 +14,7 @@ export default function ChatLayout({
   // we don't want to show the sidebar by default when the user opens the side panel
   defaultSidebarOff?: boolean;
 }) {
-  const { sidebarInitiallyVisible } = useChatContext();
+  const { sidebarInitiallyVisible, projects } = useChatContext();
 
   const [sidebarVisible, setSidebarVisible] = useState(
     (sidebarInitiallyVisible && !defaultSidebarOff) ?? false
@@ -26,14 +27,14 @@ export default function ChatLayout({
   }, []);
 
   return (
-    <>
+    <ProjectsProvider initialProjects={projects}>
       <div className="overscroll-y-contain overflow-y-scroll overscroll-contain left-0 top-0 w-full h-svh">
         <ChatPage
           toggle={toggle}
           sidebarVisible={sidebarVisible}
           firstMessage={firstMessage}
         />
       </div>
-    </>
+    </ProjectsProvider>
   );
 }

web/src/app/chat/WrappedChat.tsx

@@ -36,6 +36,7 @@ export default async function Layout({
     ccPairs,
     inputPrompts,
     proSearchToggled,
+    projects,
   } = data;
 
   return (
@@ -56,6 +57,7 @@ export default async function Layout({
           availableTools,
           shouldShowWelcomeModal,
           defaultAssistantId,
+          projects,
         }}
       >
         {children}

web/src/app/chat/layout.tsx

@@ -1,6 +1,5 @@
 import { SEARCH_PARAMS } from "@/lib/extension/constants";
 import ChatLayout from "./WrappedChat";
-import { ProjectsProvider } from "./projects/ProjectsContext";
 
 export default async function Page(props: {
   searchParams: Promise<{ [key: string]: string }>;
@@ -11,11 +10,9 @@ export default async function Page(props: {
     searchParams[SEARCH_PARAMS.DEFAULT_SIDEBAR_OFF] === "true";
 
   return (
-    <ProjectsProvider>
-      <ChatLayout
-        firstMessage={firstMessage}
-        defaultSidebarOff={defaultSidebarOff}
-      />
-    </ProjectsProvider>
+    <ChatLayout
+      firstMessage={firstMessage}
+      defaultSidebarOff={defaultSidebarOff}
+    />
   );
 }

web/src/app/chat/page.tsx

@@ -72,12 +72,14 @@ const ProjectsContext = createContext<ProjectsContextType | undefined>(
 
 interface ProjectsProviderProps {
   children: ReactNode;
+  initialProjects?: Project[];
 }
 
 export const ProjectsProvider: React.FC<ProjectsProviderProps> = ({
   children,
+  initialProjects = [],
 }) => {
-  const [projects, setProjects] = useState<Project[]>([]);
+  const [projects, setProjects] = useState<Project[]>(initialProjects);
   const [recentFiles, setRecentFiles] = useState<ProjectFile[]>([]);
   const [currentProjectDetails, setCurrentProjectDetails] =
     useState<ProjectDetails | null>(null);
@@ -261,17 +263,17 @@ export const ProjectsProvider: React.FC<ProjectsProviderProps> = ({
   );
 
   useEffect(() => {
-    // Initial load
+    // Initial load - only fetch recent files since projects come from props
     setIsLoading(true);
-    Promise.all([fetchProjects(), getRecentFiles()])
-      .then(([, recent]) => {
+    getRecentFiles()
+      .then((recent) => {
         setRecentFiles(recent);
       })
       .catch(() => {
         // errors captured in individual calls
       })
       .finally(() => setIsLoading(false));
-  }, [fetchProjects, getRecentFiles]);
+  }, [getRecentFiles]);
 
   useEffect(() => {
     if (currentProjectId) {

web/src/app/chat/projects/ProjectsContext.tsx

@@ -35,6 +35,7 @@ export default async function GalleryPage(props: {
     defaultAssistantId,
     inputPrompts,
     proSearchToggled,
+    projects,
   } = data;
 
   return (
@@ -54,6 +55,7 @@ export default async function GalleryPage(props: {
         llmProviders,
         shouldShowWelcomeModal,
         defaultAssistantId,
+        projects,
       }}
     >
       {shouldShowWelcomeModal && (

web/src/app/ee/assistants/stats/[id]/page.tsx

@@ -62,6 +62,7 @@ export async function Layout({ children }: { children: React.ReactNode }) {
     inputPrompts,
     proSearchToggled,
     availableTools,
+    projects,
   } = data;
 
   return (
@@ -81,6 +82,7 @@ export async function Layout({ children }: { children: React.ReactNode }) {
         llmProviders,
         shouldShowWelcomeModal,
         defaultAssistantId,
+        projects,
       }}
     >
       <ClientLayout

web/src/components/admin/Layout.tsx

@@ -12,6 +12,7 @@ import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces"
 import { useSearchParams } from "next/navigation";
 import { useRouter } from "next/navigation";
 import { ToolSnapshot } from "@/lib/tools/interfaces";
+import { Project } from "@/app/chat/projects/projectsService";
 
 interface ChatContextProps {
   chatSessions: ChatSession[];
@@ -33,6 +34,7 @@ interface ChatContextProps {
   refreshInputPrompts: () => Promise<void>;
   inputPrompts: InputPrompt[];
   proSearchToggled: boolean;
+  projects: Project[];
 }
 
 const ChatContext = createContext<ChatContextProps | undefined>(undefined);

web/src/components/context/ChatContext.tsx

@@ -30,6 +30,7 @@ import {
 } from "../constants";
 import { ToolSnapshot } from "../tools/interfaces";
 import { fetchToolsSS } from "../tools/fetchTools";
+import { Project } from "@/app/chat/projects/projectsService";
 
 interface FetchChatDataResult {
   user: User | null;
@@ -46,6 +47,7 @@ interface FetchChatDataResult {
   shouldShowWelcomeModal: boolean;
   inputPrompts: InputPrompt[];
   proSearchToggled: boolean;
+  projects: Project[];
 }
 
 export async function fetchChatData(searchParams: {
@@ -62,6 +64,7 @@ export async function fetchChatData(searchParams: {
     fetchLLMProvidersSS(),
     fetchSS("/input_prompt?include_public=true"),
     fetchToolsSS(),
+    fetchSS("/user/projects/"),
   ];
 
   let results: (
@@ -75,7 +78,8 @@ export async function fetchChatData(searchParams: {
     | null
     | InputPrompt[]
     | ToolSnapshot[]
-  )[] = [null, null, null, null, null, null, null, null, null, null];
+    | Project[]
+  )[] = [null, null, null, null, null, null, null, null, null, null, null];
   try {
     results = await Promise.all(tasks);
   } catch (e) {
@@ -101,6 +105,13 @@ export async function fetchChatData(searchParams: {
 
   const availableTools = (results[8] || []) as ToolSnapshot[];
 
+  let projects: Project[] = [];
+  if (results[9] instanceof Response && results[9].ok) {
+    projects = await results[9].json();
+  } else {
+    console.log("Failed to fetch projects");
+  }
+
   const authDisabled = authTypeMetadata?.authType === "disabled";
 
   // TODO Validate need
@@ -241,5 +252,6 @@ export async function fetchChatData(searchParams: {
     shouldShowWelcomeModal,
     inputPrompts,
     proSearchToggled,
+    projects,
   };
 }