From ee774dbd177231e587fc6b83c7938a633504207b Mon Sep 17 00:00:00 2001 From: Richard Kuo Date: Thu, 8 May 2025 21:44:03 -0700 Subject: [PATCH 1/2] set field size limit --- backend/onyx/connectors/salesforce/sqlite_functions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/onyx/connectors/salesforce/sqlite_functions.py b/backend/onyx/connectors/salesforce/sqlite_functions.py index f6e030efe39..2a6347e8b97 100644 --- a/backend/onyx/connectors/salesforce/sqlite_functions.py +++ b/backend/onyx/connectors/salesforce/sqlite_functions.py @@ -2,6 +2,7 @@ import json import os import sqlite3 +import sys import time from collections.abc import Iterator from pathlib import Path @@ -351,6 +352,9 @@ def update_from_csv( if self._conn is None: raise RuntimeError("Database connection is closed") + # some customers need this to be larger than the default 128KB + csv.field_size_limit(sys.maxsize) + updated_ids = [] with self._conn: From 107a250ddb2aef6bc552f70054a1d39fb01ceb2c Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Onyx)" Date: Fri, 9 May 2025 10:43:12 -0700 Subject: [PATCH 2/2] don't use sys.maxsize --- backend/onyx/connectors/salesforce/sqlite_functions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/onyx/connectors/salesforce/sqlite_functions.py b/backend/onyx/connectors/salesforce/sqlite_functions.py index 2a6347e8b97..9d314cb5d35 100644 --- a/backend/onyx/connectors/salesforce/sqlite_functions.py +++ b/backend/onyx/connectors/salesforce/sqlite_functions.py @@ -2,7 +2,6 @@ import json import os import sqlite3 -import sys import time from collections.abc import Iterator from pathlib import Path @@ -352,8 +351,8 @@ def update_from_csv( if self._conn is None: raise RuntimeError("Database connection is closed") - # some customers need this to be larger than the default 128KB - csv.field_size_limit(sys.maxsize) + # some customers need this to be larger than the default 128KB, go with 16MB + csv.field_size_limit(16 * 1024 * 1024) updated_ids = []