CW comments

Author: evan-onyx

backend/ee/onyx/server/reporting/usage_export_generation.py

@@ -67,7 +67,7 @@ def generate_chat_messages_report(
         file_id = file_store.save_file(
             content=temp_file,
             display_name=file_name,
-            file_origin=FileOrigin.OTHER,
+            file_origin=FileOrigin.GENERATED_REPORT,
             file_type="text/csv",
         )
 
@@ -99,7 +99,7 @@ def generate_user_report(
         file_id = file_store.save_file(
             content=temp_file,
             display_name=file_name,
-            file_origin=FileOrigin.OTHER,
+            file_origin=FileOrigin.GENERATED_REPORT,
             file_type="text/csv",
         )
 

backend/onyx/indexing/indexing_pipeline.py

@@ -867,13 +867,8 @@ def index_doc_batch(
         user_file_id_to_raw_text: dict[int, str] = {}
         for document_id in updatable_ids:
             # Only calculate token counts for documents that have a user file ID
-            if (
-                document_id not in doc_id_to_user_file_id
-                or doc_id_to_user_file_id[document_id] is None
-            ):
-                continue
 
-            user_file_id = doc_id_to_user_file_id[document_id]
+            user_file_id = doc_id_to_user_file_id.get(document_id)
             if user_file_id is None:
                 continue
 

backend/onyx/server/documents/connector.py

@@ -441,7 +441,9 @@ def is_zip_file(file: UploadFile) -> bool:
     )
 
 
-def upload_files(files: list[UploadFile]) -> FileUploadResponse:
+def upload_files(
+    files: list[UploadFile], file_origin: FileOrigin = FileOrigin.CONNECTOR
+) -> FileUploadResponse:
     for file in files:
         if not file.filename:
             raise HTTPException(status_code=400, detail="File name cannot be empty")
@@ -497,7 +499,7 @@ def should_process_file(file_path: str) -> bool:
                 text_file_id = file_store.save_file(
                     content=io.BytesIO(extracted_text.encode()),
                     display_name=file.filename,
-                    file_origin=FileOrigin.CHAT_UPLOAD,
+                    file_origin=file_origin,
                     file_type="text/plain",
                 )
                 deduped_file_paths.append(text_file_id)
@@ -528,7 +530,7 @@ def upload_files_api(
     files: list[UploadFile],
     _: User = Depends(current_curator_or_admin_user),
 ) -> FileUploadResponse:
-    return upload_files(files)
+    return upload_files(files, FileOrigin.OTHER)
 
 
 @router.get("/admin/connector")

backend/onyx/server/query_and_chat/chat_backend.py

@@ -716,51 +716,6 @@ def upload_files_for_chat(
                 detail="Images must be less than 20MB",
             )
 
-    # file_store = get_default_file_store()
-
-    # file_info: list[tuple[str, str | None, ChatFileType]] = []
-    # for file in files:
-    # file_type = mime_type_to_chat_file_type(file.content_type)
-
-    # file_content = file.file.read()  # Read the file content
-
-    # # NOTE: Image conversion to JPEG used to be enforced here.
-    # # This was removed to:
-    # # 1. Preserve original file content for downloads
-    # # 2. Maintain transparency in formats like PNG
-    # # 3. Ameliorate issue with file conversion
-    # file_content_io = io.BytesIO(file_content)
-
-    # new_content_type = file.content_type
-
-    # # Store the file normally
-    # file_id = file_store.save_file(
-    #     content=file_content_io,
-    #     display_name=file.filename,
-    #     file_origin=FileOrigin.CHAT_UPLOAD,
-    #     file_type=new_content_type or file_type.value,
-    # )
-
-    # # 4) If the file is a doc, extract text and store that separately
-    # if file_type == ChatFileType.DOC:
-    #     # Re-wrap bytes in a fresh BytesIO so we start at position 0
-    #     extracted_text_io = io.BytesIO(file_content)
-    #     extracted_text = extract_file_text(
-    #         file=extracted_text_io,  # use the bytes we already read
-    #         file_name=file.filename or "",
-    #     )
-
-    #     text_file_id = file_store.save_file(
-    #         content=io.BytesIO(extracted_text.encode()),
-    #         display_name=file.filename,
-    #         file_origin=FileOrigin.CHAT_UPLOAD,
-    #         file_type="text/plain",
-    #     )
-    #     # Return the text file as the "main" file descriptor for doc types
-    #     file_info.append((text_file_id, file.filename, ChatFileType.PLAIN_TEXT))
-    # else:
-    #     file_info.append((file_id, file.filename, file_type))
-
     # 5) Create a user file for each uploaded file
     user_files = create_user_files(files, RECENT_DOCS_FOLDER_ID, user, db_session)
     for user_file in user_files: