From bfb67b4eb3962523bc11833ee1c0e0153865b13c Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Wed, 9 Jul 2025 17:18:53 -0700 Subject: [PATCH 01/11] Add basic structure for frontend email connector --- web/src/app/admin/add-connector/page.tsx | 2 + .../[connector]/AddConnectorPage.tsx | 316 +++++++++--------- .../connectors/[connector]/pages/Advanced.tsx | 6 +- .../pages/DynamicConnectorCreationForm.tsx | 10 +- web/src/components/icons/icons.tsx | 6 + web/src/lib/search/interfaces.ts | 1 + web/src/lib/sources.ts | 6 + 7 files changed, 176 insertions(+), 171 deletions(-) diff --git a/web/src/app/admin/add-connector/page.tsx b/web/src/app/admin/add-connector/page.tsx index 91dccea0bab..79e0ef84e50 100644 --- a/web/src/app/admin/add-connector/page.tsx +++ b/web/src/app/admin/add-connector/page.tsx @@ -287,6 +287,8 @@ function getCategoryDescription(category: SourceCategory): string { return "Connect to cloud storage and file hosting services."; case SourceCategory.Wiki: return "Link to wiki and knowledge base platforms."; + case SourceCategory.Personal: + return "Connect to personal tools."; case SourceCategory.Other: return "Connect to other miscellaneous knowledge sources."; default: diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx index c50f5e3faf3..b531883292d 100644 --- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx +++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx @@ -498,172 +498,166 @@ export default function AddConnector({ } }} > - {(formikProps) => { - return ( -
- {popup} - - {uploading && } - - {creatingConnector && } - - } - title={displayName} - farRightElement={undefined} - /> - - {formStep == 0 && ( - - Select a credential - - {connector == ValidSources.Gmail ? ( - - ) : ( - <> - - {!createCredentialFormToggle && ( -
- {/* Button to pop up a form to manually enter credentials */} - - {/* Button to sign in via OAuth */} - {oauthSupportedSources.includes(connector) && - (NEXT_PUBLIC_CLOUD_ENABLED || - NEXT_PUBLIC_TEST_ENV) && ( - - )} -
- )} - - {createCredentialFormToggle && ( - - setCreateCredentialFormToggle(false) - } + } else { + setCreateCredentialFormToggle( + (createConnectorToggle) => !createConnectorToggle + ); + } + }} > - {oauthDetailsLoading ? ( - - ) : ( - <> - - Create a {getSourceDisplayName(connector)}{" "} - credential - - {oauthDetails && oauthDetails.oauth_enabled ? ( - - ) : ( - - setCreateCredentialFormToggle(false) - } - /> - )} - + Create New + + {/* Button to sign in via OAuth */} + {oauthSupportedSources.includes(connector) && + (NEXT_PUBLIC_CLOUD_ENABLED || NEXT_PUBLIC_TEST_ENV) && ( + )} - - )} - - )} - - )} - - {formStep == 1 && ( - - - - )} - - {formStep === 2 && ( - - - - )} - - -
- ); - }} + + )} + + {createCredentialFormToggle && ( + + setCreateCredentialFormToggle(false) + } + > + {oauthDetailsLoading ? ( + + ) : ( + <> + + Create a {getSourceDisplayName(connector)}{" "} + credential + + {oauthDetails && oauthDetails.oauth_enabled ? ( + + ) : ( + + setCreateCredentialFormToggle(false) + } + /> + )} + + )} + + )} + + )} + + )} + + {formStep == 1 && ( + + + + )} + + {formStep === 2 && ( + + + + )} + + + + )} ); } diff --git a/web/src/app/admin/connectors/[connector]/pages/Advanced.tsx b/web/src/app/admin/connectors/[connector]/pages/Advanced.tsx index b1f0128329e..79921dfe5db 100644 --- a/web/src/app/admin/connectors/[connector]/pages/Advanced.tsx +++ b/web/src/app/admin/connectors/[connector]/pages/Advanced.tsx @@ -3,7 +3,7 @@ import NumberInput from "./ConnectorInput/NumberInput"; import { TextFormField } from "@/components/Field"; import { TrashIcon } from "@/components/icons/icons"; -const AdvancedFormPage = () => { +export default function AdvancedFormPage() { return (

@@ -42,6 +42,4 @@ const AdvancedFormPage = () => {

); -}; - -export default AdvancedFormPage; +} diff --git a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx index 1dc9dd47f7c..86b3b232ba7 100644 --- a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx +++ b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx @@ -1,4 +1,4 @@ -import React, { FC, useEffect, useState } from "react"; +import React, { useEffect, useState } from "react"; import CredentialSubText from "@/components/credentials/CredentialFields"; import { ConnectionConfiguration } from "@/lib/connectors/connectors"; import { TextFormField } from "@/components/Field"; @@ -17,12 +17,12 @@ export interface DynamicConnectionFormProps { currentCredential: Credential | null; } -const DynamicConnectionForm: FC = ({ +export default function DynamicConnectionForm({ config, values, connector, currentCredential, -}) => { +}: DynamicConnectionFormProps) { const { setFieldValue } = useFormikContext(); // Get Formik's context functions const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); @@ -97,6 +97,4 @@ const DynamicConnectionForm: FC = ({ )} ); -}; - -export default DynamicConnectionForm; +} diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx index 8b5f6b30fb9..92cb416949e 100644 --- a/web/src/components/icons/icons.tsx +++ b/web/src/components/icons/icons.tsx @@ -30,6 +30,7 @@ import { FiCpu, FiInfo, FiBarChart2, + FiMail, } from "react-icons/fi"; import { SiBookstack } from "react-icons/si"; import { StaticImageData } from "next/image"; @@ -1006,6 +1007,11 @@ export const LightSettingsIcon = ({ ); }; +export const EmailIcon = ({ + size = 24, + className = defaultTailwindCSSBlue, +}: IconProps) => ; + // COMPANY LOGOS export const LoopioIcon = ({ diff --git a/web/src/lib/search/interfaces.ts b/web/src/lib/search/interfaces.ts index 1fde64f8d9e..5fe49cbcbcb 100644 --- a/web/src/lib/search/interfaces.ts +++ b/web/src/lib/search/interfaces.ts @@ -169,6 +169,7 @@ export enum SourceCategory { Messaging = "Messaging", ProjectManagement = "Project Management", CodeRepository = "Code Repository", + Personal = "Personal", Other = "Other", } diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts index ce0bd90da64..10b67fc7271 100644 --- a/web/src/lib/sources.ts +++ b/web/src/lib/sources.ts @@ -44,6 +44,7 @@ import { FileIcon2, GitbookIcon, HighspotIcon, + EmailIcon, } from "@/components/icons/icons"; import { ValidSources } from "./types"; import { SourceCategory, SourceMetadata } from "./search/interfaces"; @@ -345,6 +346,11 @@ export const SOURCE_METADATA_MAP: SourceMap = { category: SourceCategory.Wiki, docs: "https://docs.onyx.app/connectors/highspot", }, + email: { + icon: EmailIcon, + displayName: "Email", + category: SourceCategory.Personal, + }, // currently used for the Internet Search tool docs, which is why // a globe is used not_applicable: { From c2b09456ddd53bbb22c580e7b0a2130138d2e44a Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Thu, 10 Jul 2025 12:07:45 -0700 Subject: [PATCH 02/11] Update names of credentials-json keys --- backend/onyx/connectors/imap/connector.py | 10 ++++++---- .../tests/daily/connectors/imap/test_imap_connector.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/onyx/connectors/imap/connector.py b/backend/onyx/connectors/imap/connector.py index a5b8b2ef9c9..621de28b337 100644 --- a/backend/onyx/connectors/imap/connector.py +++ b/backend/onyx/connectors/imap/connector.py @@ -33,6 +33,8 @@ _DEFAULT_IMAP_PORT_NUMBER = int(os.environ.get("IMAP_PORT", 993)) _IMAP_OKAY_STATUS = "OK" _PAGE_SIZE = 100 +_USERNAME_KEY = "imap_username" +_PASSWORD_KEY = "imap_password" # An email has a list of mailboxes. @@ -102,8 +104,8 @@ def get_or_raise(name: str) -> str: if self._login_state == LoginState.LoggedIn: return - username = get_or_raise("username") - password = get_or_raise("password") + username = get_or_raise(_USERNAME_KEY) + password = get_or_raise(_PASSWORD_KEY) self._login_state = LoginState.LoggedIn self.mail_client.login(user=username, password=password) @@ -433,8 +435,8 @@ def _parse_singular_addr(raw_header: str) -> tuple[str, str]: tenant_id=None, connector_name=DocumentSource.IMAP, credential_json={ - "username": username, - "password": password, + _USERNAME_KEY: username, + _PASSWORD_KEY: password, }, ) ) diff --git a/backend/tests/daily/connectors/imap/test_imap_connector.py b/backend/tests/daily/connectors/imap/test_imap_connector.py index ec74c9043a8..9d7d3f3ca96 100644 --- a/backend/tests/daily/connectors/imap/test_imap_connector.py +++ b/backend/tests/daily/connectors/imap/test_imap_connector.py @@ -36,8 +36,8 @@ def imap_connector() -> ImapConnector: tenant_id=None, connector_name=DocumentSource.IMAP, credential_json={ - "username": username, - "password": password, + "imap_username": username, + "imap_password": password, }, ) ) From f53484abbd2bb8a561e585d6c9c4112b713bc787 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Thu, 10 Jul 2025 12:20:28 -0700 Subject: [PATCH 03/11] Fix up configurations workflow --- backend/onyx/connectors/factory.py | 2 ++ web/src/lib/connectors/connectors.tsx | 39 +++++++++++++++++++++++++++ web/src/lib/connectors/credentials.ts | 13 +++++++++ web/src/lib/sources.ts | 2 +- web/src/lib/types.ts | 1 + 5 files changed, 56 insertions(+), 1 deletion(-) diff --git a/backend/onyx/connectors/factory.py b/backend/onyx/connectors/factory.py index 6135feaa1ec..a6b743c8281 100644 --- a/backend/onyx/connectors/factory.py +++ b/backend/onyx/connectors/factory.py @@ -33,6 +33,7 @@ from onyx.connectors.guru.connector import GuruConnector from onyx.connectors.highspot.connector import HighspotConnector from onyx.connectors.hubspot.connector import HubSpotConnector +from onyx.connectors.imap.connector import ImapConnector from onyx.connectors.interfaces import BaseConnector from onyx.connectors.interfaces import CheckpointedConnector from onyx.connectors.interfaces import CredentialsConnector @@ -121,6 +122,7 @@ def identify_connector_class( DocumentSource.EGNYTE: EgnyteConnector, DocumentSource.AIRTABLE: AirtableConnector, DocumentSource.HIGHSPOT: HighspotConnector, + DocumentSource.IMAP: ImapConnector, # just for integration tests DocumentSource.MOCK_CONNECTOR: MockConnector, } diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx index 3fc6a3ed7f1..f3aceef9e71 100644 --- a/web/src/lib/connectors/connectors.tsx +++ b/web/src/lib/connectors/connectors.tsx @@ -1333,6 +1333,39 @@ For example, specifying .*-support.* as a "channel" will cause the connector to ], advanced_values: [], }, + imap: { + description: "Configure Email connector", + values: [ + { + type: "text", + query: "Enter the IMAP server host:", + label: "IMAP Server Host", + name: "host", + optional: false, + description: + "The IMAP server hostname (e.g., imap.gmail.com, outlook.office365.com)", + }, + { + type: "number", + query: "Enter the IMAP server port:", + label: "IMAP Server Port", + name: "port", + optional: true, + default: 993, + description: "The IMAP server port (default: 993 for SSL)", + }, + { + type: "list", + query: "Enter mailboxes to include:", + label: "Mailboxes", + name: "mailboxes", + optional: true, + description: + "Specify mailboxes to index (e.g., INBOX, Sent, Drafts). Leave empty to index all mailboxes.", + }, + ], + advanced_values: [], + }, }; export function createConnectorInitialValues( connector: ConfigurableSources @@ -1625,3 +1658,9 @@ export interface MediaWikiConfig extends MediaWikiBaseConfig { } export interface WikipediaConfig extends MediaWikiBaseConfig {} + +export interface ImapConfig { + host: string; + port?: number; + mailboxes?: string[]; +} diff --git a/web/src/lib/connectors/credentials.ts b/web/src/lib/connectors/credentials.ts index 99c6713508b..f355d61460e 100644 --- a/web/src/lib/connectors/credentials.ts +++ b/web/src/lib/connectors/credentials.ts @@ -244,6 +244,11 @@ export interface HighspotCredentialJson { highspot_secret: string; } +export interface ImapCredentialJson { + imap_username: string; + imap_password: string; +} + export const credentialTemplates: Record = { github: { github_access_token: "" } as GithubCredentialJson, gitlab: { @@ -400,6 +405,10 @@ export const credentialTemplates: Record = { highspot_key: "", highspot_secret: "", } as HighspotCredentialJson, + imap: { + imap_username: "", + imap_password: "", + } as ImapCredentialJson, }; export const credentialDisplayNames: Record = { @@ -482,6 +491,10 @@ export const credentialDisplayNames: Record = { // R2 account_id: "R2 Account ID", + + // IMAP + imap_username: "IMAP Username", + imap_password: "IMAP Password", r2_access_key_id: "R2 Access Key ID", r2_secret_access_key: "R2 Secret Access Key", diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts index 10b67fc7271..8d0c822da01 100644 --- a/web/src/lib/sources.ts +++ b/web/src/lib/sources.ts @@ -346,7 +346,7 @@ export const SOURCE_METADATA_MAP: SourceMap = { category: SourceCategory.Wiki, docs: "https://docs.onyx.app/connectors/highspot", }, - email: { + imap: { icon: EmailIcon, displayName: "Email", category: SourceCategory.Personal, diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 4f94ab2122e..6ce4cb7465c 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -419,6 +419,7 @@ export enum ValidSources { Airtable = "airtable", Gitbook = "gitbook", Highspot = "highspot", + Imap = "imap", // Federated Connectors FederatedSlack = "federated_slack", From 494b9a1399abc12a1f65da461d30bc814414da0f Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Thu, 10 Jul 2025 13:02:03 -0700 Subject: [PATCH 04/11] Edit logic on how `mail_client` is used - imaplib.IMAP4_SSL is supposed to be treated as an ephemeral object --- backend/onyx/connectors/imap/connector.py | 37 ++++++----------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/backend/onyx/connectors/imap/connector.py b/backend/onyx/connectors/imap/connector.py index 621de28b337..b1bd192c5a6 100644 --- a/backend/onyx/connectors/imap/connector.py +++ b/backend/onyx/connectors/imap/connector.py @@ -71,16 +71,6 @@ def __init__( self._port = port self._mailboxes = mailboxes self._credentials: dict[str, Any] | None = None - self._mail_client: imaplib.IMAP4_SSL | None = None - self._login_state: LoginState = LoginState.LoggedOut - - @property - def mail_client(self) -> imaplib.IMAP4_SSL: - if not self._mail_client: - raise RuntimeError( - "No mail-client has been initialized; call `set_credentials_provider` first" - ) - return self._mail_client @property def credentials(self) -> dict[str, Any]: @@ -90,7 +80,7 @@ def credentials(self) -> dict[str, Any]: ) return self._credentials - def _login(self) -> None: + def _login(self) -> imaplib.IMAP4_SSL: def get_or_raise(name: str) -> str: value = self.credentials.get(name) if not value: @@ -101,21 +91,16 @@ def get_or_raise(name: str) -> str: ) return value - if self._login_state == LoginState.LoggedIn: - return - username = get_or_raise(_USERNAME_KEY) password = get_or_raise(_PASSWORD_KEY) - self._login_state = LoginState.LoggedIn - self.mail_client.login(user=username, password=password) + mail_client = imaplib.IMAP4_SSL(host=self._host, port=self._port) + status, _data = mail_client.login(user=username, password=password) - def _logout(self) -> None: - if self._login_state == LoginState.LoggedOut: - return + if status != _IMAP_OKAY_STATUS: + raise RuntimeError(f"Failed to log into imap server; {status=}") - self._login_state = LoginState.LoggedOut - self.mail_client.logout() + return mail_client def _load_from_checkpoint( self, @@ -127,7 +112,7 @@ def _load_from_checkpoint( checkpoint = cast(ImapCheckpoint, copy.deepcopy(checkpoint)) checkpoint.has_more = True - self._login() + mail_client = self._login() if checkpoint.todo_mailboxes is None: # This is the dummy checkpoint. @@ -136,7 +121,7 @@ def _load_from_checkpoint( checkpoint.todo_mailboxes = _sanitize_mailbox_names(self._mailboxes) else: fetched_mailboxes = _fetch_all_mailboxes_for_email_account( - mail_client=self.mail_client + mail_client=mail_client ) if not fetched_mailboxes: raise RuntimeError( @@ -153,7 +138,7 @@ def _load_from_checkpoint( mailbox = checkpoint.todo_mailboxes.pop() checkpoint.todo_email_ids = _fetch_email_ids_in_mailbox( - mail_client=self.mail_client, + mail_client=mail_client, mailbox=mailbox, start=start, end=end, @@ -165,7 +150,7 @@ def _load_from_checkpoint( checkpoint.todo_email_ids = checkpoint.todo_email_ids[_PAGE_SIZE:] for email_id in current_todos: - email_msg = _fetch_email(mail_client=self.mail_client, email_id=email_id) + email_msg = _fetch_email(mail_client=mail_client, email_id=email_id) if not email_msg: logger.warn(f"Failed to fetch message {email_id=}; skipping") continue @@ -187,7 +172,6 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None def validate_connector_settings(self) -> None: self._login() - self._logout() # impls for CredentialsConnector @@ -195,7 +179,6 @@ def set_credentials_provider( self, credentials_provider: CredentialsProviderInterface ) -> None: self._credentials = credentials_provider.get_credentials() - self._mail_client = imaplib.IMAP4_SSL(host=self._host, port=self._port) # impls for CheckpointedConnector From 13a9e3dd0b9f41abcd7b3cfad77d5cb9070baa06 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Thu, 10 Jul 2025 13:07:36 -0700 Subject: [PATCH 05/11] Edit helper name and add docs --- backend/onyx/connectors/imap/connector.py | 24 ++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/backend/onyx/connectors/imap/connector.py b/backend/onyx/connectors/imap/connector.py index b1bd192c5a6..6985480bb41 100644 --- a/backend/onyx/connectors/imap/connector.py +++ b/backend/onyx/connectors/imap/connector.py @@ -80,7 +80,25 @@ def credentials(self) -> dict[str, Any]: ) return self._credentials - def _login(self) -> imaplib.IMAP4_SSL: + def _get_mail_client(self) -> imaplib.IMAP4_SSL: + """ + Returns a new `imaplib.IMAP4_SSL` instance. + + The `imaplib.IMAP4_SSL` object is supposed to be an "ephemeral" object; it's not something that you can login, + logout, then log back into again. I.e., the following will fail: + + ```py + mail_client.login(..) + mail_client.logout(); + mail_client.login(..) + ``` + + Therefore, you need a fresh, new instance in order to operate with IMAP. This function gives one to you. + + # Notes + This function will throw an error if the credentials have not yet been set. + """ + def get_or_raise(name: str) -> str: value = self.credentials.get(name) if not value: @@ -112,7 +130,7 @@ def _load_from_checkpoint( checkpoint = cast(ImapCheckpoint, copy.deepcopy(checkpoint)) checkpoint.has_more = True - mail_client = self._login() + mail_client = self._get_mail_client() if checkpoint.todo_mailboxes is None: # This is the dummy checkpoint. @@ -171,7 +189,7 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None raise NotImplementedError("Use `set_credentials_provider` instead") def validate_connector_settings(self) -> None: - self._login() + self._get_mail_client() # impls for CredentialsConnector From 323350f43afe8d7f6136dfacfab4419cd4deb4c3 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Thu, 10 Jul 2025 13:55:13 -0700 Subject: [PATCH 06/11] Fix invalid mailbox selection error --- backend/onyx/connectors/imap/connector.py | 38 ++++++++++++++++++----- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/backend/onyx/connectors/imap/connector.py b/backend/onyx/connectors/imap/connector.py index 6985480bb41..71e1b50fad0 100644 --- a/backend/onyx/connectors/imap/connector.py +++ b/backend/onyx/connectors/imap/connector.py @@ -12,6 +12,7 @@ from typing import cast import bs4 +from pydantic import BaseModel from onyx.access.models import ExternalAccess from onyx.configs.constants import DocumentSource @@ -37,6 +38,11 @@ _PASSWORD_KEY = "imap_password" +class CurrentMailbox(BaseModel): + mailbox: str + todo_email_ids: list[str] + + # An email has a list of mailboxes. # Each mailbox has a list of email-ids inside of it. # @@ -49,7 +55,7 @@ # For initial checkpointing, set both fields to `None`. class ImapCheckpoint(ConnectorCheckpoint): todo_mailboxes: list[str] | None = None - todo_email_ids: list[str] | None = None + current_mailbox: CurrentMailbox | None = None class LoginState(str, Enum): @@ -149,23 +155,35 @@ def _load_from_checkpoint( return checkpoint - if not checkpoint.todo_email_ids: + if ( + not checkpoint.current_mailbox + or not checkpoint.current_mailbox.todo_email_ids + ): if not checkpoint.todo_mailboxes: checkpoint.has_more = False return checkpoint mailbox = checkpoint.todo_mailboxes.pop() - checkpoint.todo_email_ids = _fetch_email_ids_in_mailbox( + email_ids = _fetch_email_ids_in_mailbox( mail_client=mail_client, mailbox=mailbox, start=start, end=end, ) + checkpoint.current_mailbox = CurrentMailbox( + mailbox=mailbox, + todo_email_ids=email_ids, + ) + _select_mailbox( + mail_client=mail_client, mailbox=checkpoint.current_mailbox.mailbox + ) current_todos = cast( - list, copy.deepcopy(checkpoint.todo_email_ids[:_PAGE_SIZE]) + list, copy.deepcopy(checkpoint.current_mailbox.todo_email_ids[:_PAGE_SIZE]) + ) + checkpoint.current_mailbox.todo_email_ids = ( + checkpoint.current_mailbox.todo_email_ids[_PAGE_SIZE:] ) - checkpoint.todo_email_ids = checkpoint.todo_email_ids[_PAGE_SIZE:] for email_id in current_todos: email_msg = _fetch_email(mail_client=mail_client, email_id=email_id) @@ -267,15 +285,19 @@ def _fetch_all_mailboxes_for_email_account(mail_client: imaplib.IMAP4_SSL) -> li return mailboxes +def _select_mailbox(mail_client: imaplib.IMAP4_SSL, mailbox: str) -> None: + status, _ids = mail_client.select(mailbox=mailbox, readonly=True) + if status != _IMAP_OKAY_STATUS: + raise RuntimeError(f"Failed to select {mailbox=}") + + def _fetch_email_ids_in_mailbox( mail_client: imaplib.IMAP4_SSL, mailbox: str, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch, ) -> list[str]: - status, _ids = mail_client.select(mailbox=mailbox, readonly=True) - if status != _IMAP_OKAY_STATUS: - raise RuntimeError(f"Failed to select {mailbox=}") + _select_mailbox(mail_client=mail_client, mailbox=mailbox) start_str = datetime.fromtimestamp(start, tz=timezone.utc).strftime("%d-%b-%Y") end_str = datetime.fromtimestamp(end, tz=timezone.utc).strftime("%d-%b-%Y") From c146c142e6e158d815c000d369370a64d9014b0f Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Fri, 11 Jul 2025 08:50:12 -0700 Subject: [PATCH 07/11] Implement greptile suggestions --- web/src/app/admin/add-connector/page.tsx | 2 -- web/src/lib/connectors/credentials.ts | 4 ++-- web/src/lib/search/interfaces.ts | 1 - web/src/lib/sources.ts | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/web/src/app/admin/add-connector/page.tsx b/web/src/app/admin/add-connector/page.tsx index 79e0ef84e50..91dccea0bab 100644 --- a/web/src/app/admin/add-connector/page.tsx +++ b/web/src/app/admin/add-connector/page.tsx @@ -287,8 +287,6 @@ function getCategoryDescription(category: SourceCategory): string { return "Connect to cloud storage and file hosting services."; case SourceCategory.Wiki: return "Link to wiki and knowledge base platforms."; - case SourceCategory.Personal: - return "Connect to personal tools."; case SourceCategory.Other: return "Connect to other miscellaneous knowledge sources."; default: diff --git a/web/src/lib/connectors/credentials.ts b/web/src/lib/connectors/credentials.ts index f355d61460e..c4f0371e771 100644 --- a/web/src/lib/connectors/credentials.ts +++ b/web/src/lib/connectors/credentials.ts @@ -491,12 +491,12 @@ export const credentialDisplayNames: Record = { // R2 account_id: "R2 Account ID", + r2_access_key_id: "R2 Access Key ID", + r2_secret_access_key: "R2 Secret Access Key", // IMAP imap_username: "IMAP Username", imap_password: "IMAP Password", - r2_access_key_id: "R2 Access Key ID", - r2_secret_access_key: "R2 Secret Access Key", // S3 aws_access_key_id: "AWS Access Key ID", diff --git a/web/src/lib/search/interfaces.ts b/web/src/lib/search/interfaces.ts index 5fe49cbcbcb..1fde64f8d9e 100644 --- a/web/src/lib/search/interfaces.ts +++ b/web/src/lib/search/interfaces.ts @@ -169,7 +169,6 @@ export enum SourceCategory { Messaging = "Messaging", ProjectManagement = "Project Management", CodeRepository = "Code Repository", - Personal = "Personal", Other = "Other", } diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts index 8d0c822da01..c9c895d30fd 100644 --- a/web/src/lib/sources.ts +++ b/web/src/lib/sources.ts @@ -349,7 +349,7 @@ export const SOURCE_METADATA_MAP: SourceMap = { imap: { icon: EmailIcon, displayName: "Email", - category: SourceCategory.Personal, + category: SourceCategory.Messaging, }, // currently used for the Internet Search tool docs, which is why // a globe is used From 0bf364178139f1f55dbea1ab0365220ef4c43055 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Fri, 11 Jul 2025 11:16:21 -0700 Subject: [PATCH 08/11] Make recipients optional and add sender to primary-owners --- backend/onyx/connectors/imap/connector.py | 9 +++++++-- backend/onyx/connectors/imap/models.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/onyx/connectors/imap/connector.py b/backend/onyx/connectors/imap/connector.py index 71e1b50fad0..8554341804a 100644 --- a/backend/onyx/connectors/imap/connector.py +++ b/backend/onyx/connectors/imap/connector.py @@ -333,14 +333,19 @@ def _convert_email_headers_and_body_into_document( email_headers: EmailHeaders, include_perm_sync: bool, ) -> Document: - _sender_name, sender_addr = _parse_singular_addr(raw_header=email_headers.sender) - parsed_recipients = _parse_addrs(raw_header=email_headers.recipients) + sender_name, sender_addr = _parse_singular_addr(raw_header=email_headers.sender) + parsed_recipients = ( + _parse_addrs(raw_header=email_headers.recipients) + if email_headers.recipients + else [] + ) email_body = _parse_email_body(email_msg=email_msg, email_headers=email_headers) primary_owners = [ BasicExpertInfo(display_name=recipient_name, email=recipient_addr) for recipient_name, recipient_addr in parsed_recipients ] + primary_owners.append(BasicExpertInfo(display_name=sender_name, email=sender_addr)) external_access = ( ExternalAccess( external_user_emails=set(addr for _name, addr in parsed_recipients), diff --git a/backend/onyx/connectors/imap/models.py b/backend/onyx/connectors/imap/models.py index d07ee9bd202..83d59f75620 100644 --- a/backend/onyx/connectors/imap/models.py +++ b/backend/onyx/connectors/imap/models.py @@ -25,7 +25,7 @@ class EmailHeaders(BaseModel): id: str subject: str sender: str - recipients: str + recipients: str | None date: datetime @classmethod From 1367731ec69f6c229f42102aadb2e50539ea4507 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Fri, 11 Jul 2025 11:47:54 -0700 Subject: [PATCH 09/11] Add sender to external-access too; perform dedupe-ing of emails --- backend/onyx/connectors/imap/connector.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/backend/onyx/connectors/imap/connector.py b/backend/onyx/connectors/imap/connector.py index 8554341804a..8396e412ad9 100644 --- a/backend/onyx/connectors/imap/connector.py +++ b/backend/onyx/connectors/imap/connector.py @@ -340,15 +340,26 @@ def _convert_email_headers_and_body_into_document( else [] ) - email_body = _parse_email_body(email_msg=email_msg, email_headers=email_headers) - primary_owners = [ - BasicExpertInfo(display_name=recipient_name, email=recipient_addr) + expert_info_map = { + recipient_addr: BasicExpertInfo( + display_name=recipient_name, email=recipient_addr + ) for recipient_name, recipient_addr in parsed_recipients - ] - primary_owners.append(BasicExpertInfo(display_name=sender_name, email=sender_addr)) + } + if sender_addr not in expert_info_map: + expert_info_map[sender_addr] = BasicExpertInfo( + display_name=sender_name, email=sender_addr + ) + + primary_owners = list(expert_info_map.values()) + email_body = _parse_email_body(email_msg=email_msg, email_headers=email_headers) external_access = ( ExternalAccess( - external_user_emails=set(addr for _name, addr in parsed_recipients), + external_user_emails=set( + expert_info.email + for _name, expert_info in expert_info_map.items() + if expert_info.email + ), external_user_group_ids=set(), is_public=False, ) From 172dc056eba243f1d672764bee9b7699cc68e929 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Fri, 11 Jul 2025 11:51:40 -0700 Subject: [PATCH 10/11] Simplify logic --- backend/onyx/connectors/imap/connector.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/backend/onyx/connectors/imap/connector.py b/backend/onyx/connectors/imap/connector.py index 8396e412ad9..c8f28ce6226 100644 --- a/backend/onyx/connectors/imap/connector.py +++ b/backend/onyx/connectors/imap/connector.py @@ -351,15 +351,11 @@ def _convert_email_headers_and_body_into_document( display_name=sender_name, email=sender_addr ) - primary_owners = list(expert_info_map.values()) email_body = _parse_email_body(email_msg=email_msg, email_headers=email_headers) + primary_owners = list(expert_info_map.values()) external_access = ( ExternalAccess( - external_user_emails=set( - expert_info.email - for _name, expert_info in expert_info_map.items() - if expert_info.email - ), + external_user_emails=set(expert_info_map.keys()), external_user_group_ids=set(), is_public=False, ) From d43a512b8bab611fad7be553ec657954c2ac2378 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Mon, 14 Jul 2025 09:47:44 -0700 Subject: [PATCH 11/11] Fix imap tests --- backend/tests/daily/connectors/imap/test_imap_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/tests/daily/connectors/imap/test_imap_connector.py b/backend/tests/daily/connectors/imap/test_imap_connector.py index 9d7d3f3ca96..34249fab560 100644 --- a/backend/tests/daily/connectors/imap/test_imap_connector.py +++ b/backend/tests/daily/connectors/imap/test_imap_connector.py @@ -51,12 +51,12 @@ def imap_connector() -> ImapConnector: [ EmailDoc( subject="Testing", - recipients=set(["admin@onyx-test.com"]), + recipients=set(["admin@onyx-test.com", "raunak@onyx.app"]), body="Hello, testing.", ), EmailDoc( subject="Hello world", - recipients=set(["admin@onyx-test.com", "r@rabh.io"]), + recipients=set(["admin@onyx-test.com", "r@rabh.io", "raunak@onyx.app"]), body='Hello world, this is an email that contains multiple "To" recipients.', ), ]