From a317b115da9a36498802c780386301bf591f544c Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Mon, 9 Jun 2025 10:04:56 -0700 Subject: [PATCH 1/5] Add rate-limiting to Teams API request --- backend/onyx/connectors/teams/connector.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/onyx/connectors/teams/connector.py b/backend/onyx/connectors/teams/connector.py index 8b008047f91..ebcda9efd0f 100644 --- a/backend/onyx/connectors/teams/connector.py +++ b/backend/onyx/connectors/teams/connector.py @@ -16,6 +16,7 @@ from onyx.configs.constants import DocumentSource from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc +from onyx.connectors.cross_connector_utils.rate_limit_wrapper import rate_limit_builder from onyx.connectors.exceptions import ConnectorValidationError from onyx.connectors.exceptions import CredentialExpiredError from onyx.connectors.exceptions import InsufficientPermissionsError @@ -492,7 +493,12 @@ def _collect_documents_for_channel( continue try: - replies = list(message.replies.get_all().execute_query()) + + @rate_limit_builder(max_calls=50, period=60) + def fetch_replies() -> list[ChatMessage]: + return list(message.replies.get_all().execute_query()) + + replies = fetch_replies() thread = [message] thread.extend(replies[::-1]) From eb32025829e79ca32eb7c7a558cf396fbfa80392 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Mon, 9 Jun 2025 15:47:02 -0700 Subject: [PATCH 2/5] Add comment for rate-limiting --- backend/onyx/connectors/teams/connector.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/onyx/connectors/teams/connector.py b/backend/onyx/connectors/teams/connector.py index ebcda9efd0f..3d7b37df83e 100644 --- a/backend/onyx/connectors/teams/connector.py +++ b/backend/onyx/connectors/teams/connector.py @@ -493,8 +493,15 @@ def _collect_documents_for_channel( continue try: - - @rate_limit_builder(max_calls=50, period=60) + # Rate limit parameters obtained from: + # https://learn.microsoft.com/en-us/microsoftteams/platform/bots/how-to/rate-limit + # + # "Get Conversation" goes from (14req per 1sec) to (3600req per 3600sec) [aka, (14 req/s) down to (1 req/s)]. + # I elected to choose something in the middle (120 req per 30 sec) [4 req/s]. + MAX_CALLS = 120 + PERIOD = 30 + + @rate_limit_builder(max_calls=MAX_CALLS, period=PERIOD) def fetch_replies() -> list[ChatMessage]: return list(message.replies.get_all().execute_query()) From faa78b804598452f89ab6e3cfa9725211a293aef Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Mon, 9 Jun 2025 16:47:07 -0700 Subject: [PATCH 3/5] Implement rate-limiting for office365 library. --- backend/onyx/connectors/teams/connector.py | 63 +++++++++++++++++----- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/backend/onyx/connectors/teams/connector.py b/backend/onyx/connectors/teams/connector.py index 3d7b37df83e..dc497ac66a0 100644 --- a/backend/onyx/connectors/teams/connector.py +++ b/backend/onyx/connectors/teams/connector.py @@ -1,5 +1,6 @@ import copy import os +import time from collections.abc import Iterator from datetime import datetime from datetime import timezone @@ -16,7 +17,6 @@ from onyx.configs.constants import DocumentSource from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc -from onyx.connectors.cross_connector_utils.rate_limit_wrapper import rate_limit_builder from onyx.connectors.exceptions import ConnectorValidationError from onyx.connectors.exceptions import CredentialExpiredError from onyx.connectors.exceptions import InsufficientPermissionsError @@ -493,19 +493,54 @@ def _collect_documents_for_channel( continue try: - # Rate limit parameters obtained from: - # https://learn.microsoft.com/en-us/microsoftteams/platform/bots/how-to/rate-limit - # - # "Get Conversation" goes from (14req per 1sec) to (3600req per 3600sec) [aka, (14 req/s) down to (1 req/s)]. - # I elected to choose something in the middle (120 req per 30 sec) [4 req/s]. - MAX_CALLS = 120 - PERIOD = 30 - - @rate_limit_builder(max_calls=MAX_CALLS, period=PERIOD) - def fetch_replies() -> list[ChatMessage]: - return list(message.replies.get_all().execute_query()) - - replies = fetch_replies() + MAX_RETRIES = 10 + retries = 0 + replies: list[ChatMessage] | None = None + cre: ClientRequestException | None = None + + while True: + if retries == MAX_RETRIES: + break + + try: + replies = list(message.replies.get_all().execute_query()) + cre = None + break + except ClientRequestException as e: + cre = e + + if not cre.response: + continue + if cre.response.status_code != 429: + continue + + retry_after = int(cre.response.headers.get("Retry-After", 10)) + time.sleep(retry_after) + retries += 1 + + if cre: + failure_message = f"Retrieval of message and its replies failed; {channel.id=} {message.id}" + if cre.response: + failure_message = f"{failure_message}; {cre.response.status_code=}" + + yield ConnectorFailure( + failed_entity=EntityFailure( + entity_id=message.id, + ), + failure_message=f"Retrieval of message and its replies failed; {channel.id=} {message.id}", + exception=cre, + ) + + if not replies: + yield ConnectorFailure( + failed_entity=EntityFailure( + entity_id=message.id, + ), + failure_message=f"Retrieval of message and its replies failed; {channel.id=} {message.id}", + exception=cre, + ) + continue + thread = [message] thread.extend(replies[::-1]) From 2270e79b25ca9a17253d7e977a30b2fc2bfde8c0 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Tue, 10 Jun 2025 11:25:45 -0700 Subject: [PATCH 4/5] Remove hardcoded value --- backend/onyx/connectors/teams/connector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/onyx/connectors/teams/connector.py b/backend/onyx/connectors/teams/connector.py index dc497ac66a0..8f22aca5e35 100644 --- a/backend/onyx/connectors/teams/connector.py +++ b/backend/onyx/connectors/teams/connector.py @@ -4,6 +4,7 @@ from collections.abc import Iterator from datetime import datetime from datetime import timezone +from http import HTTPStatus from typing import Any from typing import cast @@ -511,7 +512,7 @@ def _collect_documents_for_channel( if not cre.response: continue - if cre.response.status_code != 429: + if cre.response.status_code != int(HTTPStatus.TOO_MANY_REQUESTS): continue retry_after = int(cre.response.headers.get("Retry-After", 10)) From 0d96ec0db59d7fb6cce4fb4fca463ef00b8b2f93 Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Tue, 10 Jun 2025 13:09:51 -0700 Subject: [PATCH 5/5] Fix nits on PR --- backend/onyx/connectors/teams/connector.py | 31 +++++++--------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/backend/onyx/connectors/teams/connector.py b/backend/onyx/connectors/teams/connector.py index 8f22aca5e35..e7650619e74 100644 --- a/backend/onyx/connectors/teams/connector.py +++ b/backend/onyx/connectors/teams/connector.py @@ -499,45 +499,32 @@ def _collect_documents_for_channel( replies: list[ChatMessage] | None = None cre: ClientRequestException | None = None - while True: - if retries == MAX_RETRIES: - break - + while retries < MAX_RETRIES: try: replies = list(message.replies.get_all().execute_query()) cre = None break + except ClientRequestException as e: cre = e if not cre.response: - continue + break if cre.response.status_code != int(HTTPStatus.TOO_MANY_REQUESTS): - continue + break retry_after = int(cre.response.headers.get("Retry-After", 10)) time.sleep(retry_after) retries += 1 - if cre: - failure_message = f"Retrieval of message and its replies failed; {channel.id=} {message.id}" - if cre.response: + if cre or not replies: + failure_message = f"Retrieval of message and its replies failed; {channel.id=} {message.id=}" + if cre and cre.response: failure_message = f"{failure_message}; {cre.response.status_code=}" yield ConnectorFailure( - failed_entity=EntityFailure( - entity_id=message.id, - ), - failure_message=f"Retrieval of message and its replies failed; {channel.id=} {message.id}", - exception=cre, - ) - - if not replies: - yield ConnectorFailure( - failed_entity=EntityFailure( - entity_id=message.id, - ), - failure_message=f"Retrieval of message and its replies failed; {channel.id=} {message.id}", + failed_entity=EntityFailure(entity_id=message.id), + failure_message=failure_message, exception=cre, ) continue