From c1943d3bfd9d3846fa5641a45eae851970ee4b26 Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Onyx)" Date: Tue, 29 Apr 2025 21:49:30 -0700 Subject: [PATCH 1/3] add sendgrid as option --- backend/onyx/auth/email_utils.py | 136 +++++++++++++++++---- backend/onyx/chat/process_message.py | 16 +-- backend/onyx/configs/app_configs.py | 4 +- backend/requirements/default.txt | 3 + backend/tests/unit/onyx/auth/test_email.py | 36 ++++++ 5 files changed, 165 insertions(+), 30 deletions(-) create mode 100644 backend/tests/unit/onyx/auth/test_email.py diff --git a/backend/onyx/auth/email_utils.py b/backend/onyx/auth/email_utils.py index 9c8fba65d1d..65d76349f56 100644 --- a/backend/onyx/auth/email_utils.py +++ b/backend/onyx/auth/email_utils.py @@ -1,3 +1,4 @@ +import base64 import smtplib from datetime import datetime from email.mime.image import MIMEImage @@ -6,8 +7,21 @@ from email.utils import formatdate from email.utils import make_msgid +import sendgrid # type: ignore +from sendgrid.helpers.mail import Attachment +from sendgrid.helpers.mail import Content +from sendgrid.helpers.mail import ContentId +from sendgrid.helpers.mail import Disposition +from sendgrid.helpers.mail import Email +from sendgrid.helpers.mail import FileContent +from sendgrid.helpers.mail import FileName +from sendgrid.helpers.mail import FileType +from sendgrid.helpers.mail import Mail +from sendgrid.helpers.mail import To + from onyx.configs.app_configs import EMAIL_CONFIGURED from onyx.configs.app_configs import EMAIL_FROM +from onyx.configs.app_configs import SENDGRID_API_KEY from onyx.configs.app_configs import SMTP_PASS from onyx.configs.app_configs import SMTP_PORT from onyx.configs.app_configs import SMTP_SERVER @@ -19,10 +33,12 @@ from onyx.db.models import User from onyx.server.runtime.onyx_runtime import OnyxRuntime from onyx.utils.file import FileWithMimeType +from onyx.utils.logger import setup_logger from onyx.utils.url import add_url_params from onyx.utils.variable_functionality import fetch_versioned_implementation from shared_configs.configs import MULTI_TENANT +logger = setup_logger() HTML_EMAIL_TEMPLATE = """\ @@ -176,6 +192,73 @@ def send_email( if not EMAIL_CONFIGURED: raise ValueError("Email is not configured.") + if SENDGRID_API_KEY: + send_email_with_sendgrid( + user_email, subject, html_body, text_body, mail_from, inline_png + ) + return + + send_email_with_smtplib( + user_email, subject, html_body, text_body, mail_from, inline_png + ) + + +def send_email_with_sendgrid( + user_email: str, + subject: str, + html_body: str, + text_body: str, + mail_from: str = EMAIL_FROM, + inline_png: tuple[str, bytes] | None = None, +) -> None: + from_email = Email(mail_from) if mail_from else Email("noreply@onyx.app") + to_email = To(user_email) # Change to your recipient + + mail = Mail( + from_email=from_email, + to_emails=to_email, + subject=subject, + plain_text_content=Content("text/plain", text_body), + ) + + # Add HTML content + mail.add_content(Content("text/html", html_body)) + + if inline_png: + image_name, image_data = inline_png + + # Create attachment + encoded_image = base64.b64encode(image_data).decode() + attachment = Attachment() + attachment.file_content = FileContent(encoded_image) + attachment.file_name = FileName(image_name) + attachment.file_type = FileType("image/png") + attachment.disposition = Disposition("inline") + attachment.content_id = ContentId(image_name) + + mail.add_attachment(attachment) + + # Get a JSON-ready representation of the Mail object + mail_json = mail.get() + + try: + sg = sendgrid.SendGridAPIClient(api_key=SENDGRID_API_KEY) + response = sg.client.mail.send.post(request_body=mail_json) + if response.status_code != 202: + logger.warning(f"Unexpected status code {response.status_code}") + except Exception as e: + raise e + + +def send_email_with_smtplib( + user_email: str, + subject: str, + html_body: str, + text_body: str, + mail_from: str = EMAIL_FROM, + inline_png: tuple[str, bytes] | None = None, +) -> None: + # Create a multipart/alternative message - this indicates these are alternative versions of the same content msg = MIMEMultipart("alternative") msg["Subject"] = subject @@ -264,27 +347,13 @@ def send_subscription_cancellation_email(user_email: str) -> None: ) -def send_user_email_invite( - user_email: str, current_user: User, auth_type: AuthType -) -> None: - onyx_file: FileWithMimeType | None = None - - try: - load_runtime_settings_fn = fetch_versioned_implementation( - "onyx.server.enterprise_settings.store", "load_runtime_settings" - ) - settings = load_runtime_settings_fn() - application_name = settings.application_name - except ModuleNotFoundError: - application_name = ONYX_DEFAULT_APPLICATION_NAME - - onyx_file = OnyxRuntime.get_emailable_logo() - - subject = f"Invitation to Join {application_name} Organization" +def build_user_email_invite( + from_email: str, to_email: str, application_name: str, auth_type: AuthType +) -> tuple[str, str]: heading = "You've Been Invited!" # the exact action taken by the user, and thus the message, depends on the auth type - message = f"

You have been invited by {current_user.email} to join an organization on {application_name}.

" + message = f"

You have been invited by {from_email} to join an organization on {application_name}.

" if auth_type == AuthType.CLOUD: message += ( "

To join the organization, please click the button below to set a password " @@ -309,7 +378,7 @@ def send_user_email_invite( raise ValueError(f"Invalid auth type: {auth_type}") cta_text = "Join Organization" - cta_link = f"{WEB_DOMAIN}/auth/signup?email={user_email}" + cta_link = f"{WEB_DOMAIN}/auth/signup?email={to_email}" html_content = build_html_email( application_name, @@ -322,13 +391,38 @@ def send_user_email_invite( # text content is the fallback for clients that don't support HTML # not as critical, so not having special cases for each auth type text_content = ( - f"You have been invited by {current_user.email} to join an organization on {application_name}.\n" + f"You have been invited by {from_email} to join an organization on {application_name}.\n" "To join the organization, please visit the following link:\n" - f"{WEB_DOMAIN}/auth/signup?email={user_email}\n" + f"{WEB_DOMAIN}/auth/signup?email={to_email}\n" ) if auth_type == AuthType.CLOUD: text_content += "You'll be asked to set a password or login with Google to complete your registration." + return text_content, html_content + + +def send_user_email_invite( + user_email: str, current_user: User, auth_type: AuthType +) -> None: + onyx_file: FileWithMimeType | None = None + + try: + load_runtime_settings_fn = fetch_versioned_implementation( + "onyx.server.enterprise_settings.store", "load_runtime_settings" + ) + settings = load_runtime_settings_fn() + application_name = settings.application_name + except ModuleNotFoundError: + application_name = ONYX_DEFAULT_APPLICATION_NAME + + onyx_file = OnyxRuntime.get_emailable_logo() + + subject = f"Invitation to Join {application_name} Organization" + + text_content, html_content = build_user_email_invite( + current_user.email, user_email, application_name, auth_type + ) + send_email( user_email, subject, diff --git a/backend/onyx/chat/process_message.py b/backend/onyx/chat/process_message.py index 2f6db6da65c..4ca4002a36b 100644 --- a/backend/onyx/chat/process_message.py +++ b/backend/onyx/chat/process_message.py @@ -211,13 +211,13 @@ def _handle_search_tool_response_summary( user_files: list[UserFile] | None = None, loaded_user_files: list[InMemoryChatFile] | None = None, ) -> tuple[QADocsResponse, list[DbSearchDoc], list[int] | None]: - response_sumary = cast(SearchResponseSummary, packet.response) + response_summary = cast(SearchResponseSummary, packet.response) is_extended = isinstance(packet, ExtendedToolResponse) dropped_inds = None if not selected_search_docs: - top_docs = chunks_or_sections_to_search_docs(response_sumary.top_sections) + top_docs = chunks_or_sections_to_search_docs(response_summary.top_sections) deduped_docs = top_docs if ( @@ -264,13 +264,13 @@ def _handle_search_tool_response_summary( level, question_num = packet.level, packet.level_question_num return ( QADocsResponse( - rephrased_query=response_sumary.rephrased_query, + rephrased_query=response_summary.rephrased_query, top_documents=response_docs, - predicted_flow=response_sumary.predicted_flow, - predicted_search=response_sumary.predicted_search, - applied_source_filters=response_sumary.final_filters.source_type, - applied_time_cutoff=response_sumary.final_filters.time_cutoff, - recency_bias_multiplier=response_sumary.recency_bias_multiplier, + predicted_flow=response_summary.predicted_flow, + predicted_search=response_summary.predicted_search, + applied_source_filters=response_summary.final_filters.source_type, + applied_time_cutoff=response_summary.final_filters.time_cutoff, + recency_bias_multiplier=response_summary.recency_bias_multiplier, level=level, level_question_num=question_num, ), diff --git a/backend/onyx/configs/app_configs.py b/backend/onyx/configs/app_configs.py index 13070bf5636..26241566ad0 100644 --- a/backend/onyx/configs/app_configs.py +++ b/backend/onyx/configs/app_configs.py @@ -120,9 +120,11 @@ SMTP_PORT = int(os.environ.get("SMTP_PORT") or "587") SMTP_USER = os.environ.get("SMTP_USER", "your-email@gmail.com") SMTP_PASS = os.environ.get("SMTP_PASS", "your-gmail-password") -EMAIL_CONFIGURED = all([SMTP_SERVER, SMTP_USER, SMTP_PASS]) EMAIL_FROM = os.environ.get("EMAIL_FROM") or SMTP_USER +SENDGRID_API_KEY = os.environ.get("SENDGRID_API_KEY") or "" +EMAIL_CONFIGURED = all([SMTP_SERVER, SMTP_USER, SMTP_PASS]) or SENDGRID_API_KEY + # If set, Onyx will listen to the `expires_at` returned by the identity # provider (e.g. Okta, Google, etc.) and force the user to re-authenticate # after this time has elapsed. Disabled since by default many auth providers diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt index e45042f9e33..052ef578460 100644 --- a/backend/requirements/default.txt +++ b/backend/requirements/default.txt @@ -99,3 +99,6 @@ sentry-sdk==2.14.0 prometheus_client==0.21.0 fastapi-limiter==0.1.6 prometheus_fastapi_instrumentator==7.1.0 +sendgrid==6.11.0 +python-http-client==3.3.7 +starkbank-ecdsa==2.2.0 diff --git a/backend/tests/unit/onyx/auth/test_email.py b/backend/tests/unit/onyx/auth/test_email.py new file mode 100644 index 00000000000..2f26c004441 --- /dev/null +++ b/backend/tests/unit/onyx/auth/test_email.py @@ -0,0 +1,36 @@ +import pytest + +from onyx.auth.email_utils import build_user_email_invite +from onyx.auth.email_utils import send_email +from onyx.configs.constants import AuthType +from onyx.configs.constants import ONYX_DEFAULT_APPLICATION_NAME +from onyx.db.engine import SqlEngine +from onyx.server.runtime.onyx_runtime import OnyxRuntime + + +@pytest.mark.skip( + reason="This sends real emails, so only run when you really want to test this!" +) +def test_send_user_email_invite() -> None: + SqlEngine.init_engine(pool_size=20, max_overflow=5) + + application_name = ONYX_DEFAULT_APPLICATION_NAME + + onyx_file = OnyxRuntime.get_emailable_logo() + + subject = f"Invitation to Join {application_name} Organization" + + FROM_EMAIL = "noreply@onyx.app" + TO_EMAIL = "support@onyx.app" + text_content, html_content = build_user_email_invite( + FROM_EMAIL, TO_EMAIL, ONYX_DEFAULT_APPLICATION_NAME, AuthType.CLOUD + ) + + send_email( + TO_EMAIL, + subject, + html_content, + text_content, + mail_from=FROM_EMAIL, + inline_png=("logo.png", onyx_file.data), + ) From 54dd5e94965c46770c8bc8c1f296b06296aa3c8c Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Onyx)" Date: Tue, 29 Apr 2025 22:52:59 -0700 Subject: [PATCH 2/3] code review --- backend/onyx/auth/email_utils.py | 27 +++++++++------------------ backend/requirements/default.txt | 2 -- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/backend/onyx/auth/email_utils.py b/backend/onyx/auth/email_utils.py index 65d76349f56..4fec1682ecd 100644 --- a/backend/onyx/auth/email_utils.py +++ b/backend/onyx/auth/email_utils.py @@ -32,7 +32,6 @@ from onyx.configs.constants import ONYX_SLACK_URL from onyx.db.models import User from onyx.server.runtime.onyx_runtime import OnyxRuntime -from onyx.utils.file import FileWithMimeType from onyx.utils.logger import setup_logger from onyx.utils.url import add_url_params from onyx.utils.variable_functionality import fetch_versioned_implementation @@ -212,7 +211,7 @@ def send_email_with_sendgrid( inline_png: tuple[str, bytes] | None = None, ) -> None: from_email = Email(mail_from) if mail_from else Email("noreply@onyx.app") - to_email = To(user_email) # Change to your recipient + to_email = To(user_email) mail = Mail( from_email=from_email, @@ -241,13 +240,10 @@ def send_email_with_sendgrid( # Get a JSON-ready representation of the Mail object mail_json = mail.get() - try: - sg = sendgrid.SendGridAPIClient(api_key=SENDGRID_API_KEY) - response = sg.client.mail.send.post(request_body=mail_json) - if response.status_code != 202: - logger.warning(f"Unexpected status code {response.status_code}") - except Exception as e: - raise e + sg = sendgrid.SendGridAPIClient(api_key=SENDGRID_API_KEY) + response = sg.client.mail.send.post(request_body=mail_json) # can raise + if response.status_code != 202: + logger.warning(f"Unexpected status code {response.status_code}") def send_email_with_smtplib( @@ -293,13 +289,10 @@ def send_email_with_smtplib( html_part = MIMEText(html_body, "html") msg.attach(html_part) - try: - with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as s: - s.starttls() - s.login(SMTP_USER, SMTP_PASS) - s.send_message(msg) - except Exception as e: - raise e + with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as s: + s.starttls() + s.login(SMTP_USER, SMTP_PASS) + s.send_message(msg) def send_subscription_cancellation_email(user_email: str) -> None: @@ -404,8 +397,6 @@ def build_user_email_invite( def send_user_email_invite( user_email: str, current_user: User, auth_type: AuthType ) -> None: - onyx_file: FileWithMimeType | None = None - try: load_runtime_settings_fn = fetch_versioned_implementation( "onyx.server.enterprise_settings.store", "load_runtime_settings" diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt index 052ef578460..61cd770ce3e 100644 --- a/backend/requirements/default.txt +++ b/backend/requirements/default.txt @@ -100,5 +100,3 @@ prometheus_client==0.21.0 fastapi-limiter==0.1.6 prometheus_fastapi_instrumentator==7.1.0 sendgrid==6.11.0 -python-http-client==3.3.7 -starkbank-ecdsa==2.2.0 From 1a63c391c09f926c738adddb32bf1d44c1f9b142 Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Onyx)" Date: Tue, 29 Apr 2025 23:34:30 -0700 Subject: [PATCH 3/3] mypy --- backend/onyx/auth/email_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/onyx/auth/email_utils.py b/backend/onyx/auth/email_utils.py index 4fec1682ecd..58deb0ae824 100644 --- a/backend/onyx/auth/email_utils.py +++ b/backend/onyx/auth/email_utils.py @@ -8,7 +8,7 @@ from email.utils import make_msgid import sendgrid # type: ignore -from sendgrid.helpers.mail import Attachment +from sendgrid.helpers.mail import Attachment # type: ignore from sendgrid.helpers.mail import Content from sendgrid.helpers.mail import ContentId from sendgrid.helpers.mail import Disposition