From 966ddbccc852e362d0ccbba31ec5cd629628c16b Mon Sep 17 00:00:00 2001 From: justin-tahara Date: Tue, 26 Aug 2025 14:05:05 -0700 Subject: [PATCH] --amend """ Manual approach: Generate URLs for all your projects to check OAuth clients. Requires no additional installations - just built-in Python libraries. """ def main(): print("Since we can't easily list projects without gcloud,") print("let's generate URLs manually.") print() # Ask user for project IDs print("Enter your GCP project IDs (one per line, press Enter twice when done):") projects = [] while True: project = input().strip() if not project: if projects: # If we have at least one project and get empty line break else: continue projects.append(project) if not projects: print("No projects entered.") return print(f"\nFound {len(projects)} projects.") print("Here are the direct URLs to check for OAuth 2.0 Client IDs:") print() # Generate URLs for each project for project in projects: url = f"https://console.cloud.google.com/apis/credentials?project={project}" print(f"Project: {project}") print(f"URL: {url}") print() # Save to file with open('oauth_check_urls.txt', 'w') as f: f.write("OAuth Client Check URLs\n") f.write("=" * 30 + "\n\n") for project in projects: url = f"https://console.cloud.google.com/apis/credentials?project={project}" f.write(f"Project: {project}\n") f.write(f"URL: {url}\n\n") print(f"URLs also saved to: oauth_check_urls.txt") print("\nInstructions:") --- .../onyx/background/celery/configs/base.py | 3 +- backend/onyx/configs/app_configs.py | 4 ++ backend/onyx/redis/iam_auth.py | 47 +++++++++++++++++++ backend/onyx/redis/redis_pool.py | 44 ++++++++++++++++- 4 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 backend/onyx/redis/iam_auth.py diff --git a/backend/onyx/background/celery/configs/base.py b/backend/onyx/background/celery/configs/base.py index a33c1284a5d..be5cf78c4bc 100644 --- a/backend/onyx/background/celery/configs/base.py +++ b/backend/onyx/background/celery/configs/base.py @@ -12,13 +12,14 @@ from onyx.configs.app_configs import REDIS_SSL from onyx.configs.app_configs import REDIS_SSL_CA_CERTS from onyx.configs.app_configs import REDIS_SSL_CERT_REQS +from onyx.configs.app_configs import USE_REDIS_IAM_AUTH from onyx.configs.constants import OnyxCeleryPriority from onyx.configs.constants import REDIS_SOCKET_KEEPALIVE_OPTIONS CELERY_SEPARATOR = ":" CELERY_PASSWORD_PART = "" -if REDIS_PASSWORD: +if REDIS_PASSWORD and not USE_REDIS_IAM_AUTH: CELERY_PASSWORD_PART = ":" + urllib.parse.quote(REDIS_PASSWORD, safe="") + "@" REDIS_SCHEME = "redis" diff --git a/backend/onyx/configs/app_configs.py b/backend/onyx/configs/app_configs.py index cbc33d1f6c8..278b406c2d7 100644 --- a/backend/onyx/configs/app_configs.py +++ b/backend/onyx/configs/app_configs.py @@ -231,8 +231,12 @@ except ValueError: POSTGRES_POOL_RECYCLE = POSTGRES_POOL_RECYCLE_DEFAULT +# RDS IAM authentication - enables IAM-based authentication for PostgreSQL USE_IAM_AUTH = os.getenv("USE_IAM_AUTH", "False").lower() == "true" +# Redis IAM authentication - enables IAM-based authentication for Redis ElastiCache +# Note: This is separate from RDS IAM auth as they use different authentication mechanisms +USE_REDIS_IAM_AUTH = os.getenv("USE_REDIS_IAM_AUTH", "False").lower() == "true" REDIS_SSL = os.getenv("REDIS_SSL", "").lower() == "true" REDIS_HOST = os.environ.get("REDIS_HOST") or "localhost" diff --git a/backend/onyx/redis/iam_auth.py b/backend/onyx/redis/iam_auth.py new file mode 100644 index 00000000000..8c92640b6f5 --- /dev/null +++ b/backend/onyx/redis/iam_auth.py @@ -0,0 +1,47 @@ +""" +Redis IAM Authentication Module + +This module provides Redis IAM authentication functionality for AWS ElastiCache. +Unlike RDS IAM auth, Redis IAM auth relies on IAM roles and policies rather than +generating authentication tokens. + +Key functions: +- configure_redis_iam_auth: Configure Redis connection parameters for IAM auth +- create_redis_ssl_context_if_iam: Create SSL context for secure connections +""" + +import ssl +from typing import Any + + +def configure_redis_iam_auth(connection_kwargs: dict[str, Any]) -> None: + """ + Configure Redis connection parameters for IAM authentication. + + Modifies the connection_kwargs dict in-place to: + 1. Remove password (not needed with IAM) + 2. Enable SSL with system CA certificates + 3. Set proper SSL context for secure connections + """ + # Remove password as it's not needed with IAM authentication + if "password" in connection_kwargs: + del connection_kwargs["password"] + + # Ensure SSL is enabled for IAM authentication + connection_kwargs["ssl"] = True + + # Create SSL context using system CA certificates by default + # This works with AWS ElastiCache without requiring additional CA files + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = True + ssl_context.verify_mode = ssl.CERT_REQUIRED + connection_kwargs["ssl_context"] = ssl_context + + +def create_redis_ssl_context_if_iam() -> ssl.SSLContext: + """Create an SSL context for Redis IAM authentication using system CA certificates.""" + # Use system CA certificates by default - no need for additional CA files + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = True + ssl_context.verify_mode = ssl.CERT_REQUIRED + return ssl_context diff --git a/backend/onyx/redis/redis_pool.py b/backend/onyx/redis/redis_pool.py index 584def16a5d..6e41e5be43f 100644 --- a/backend/onyx/redis/redis_pool.py +++ b/backend/onyx/redis/redis_pool.py @@ -25,8 +25,11 @@ from onyx.configs.app_configs import REDIS_SSL from onyx.configs.app_configs import REDIS_SSL_CA_CERTS from onyx.configs.app_configs import REDIS_SSL_CERT_REQS +from onyx.configs.app_configs import USE_REDIS_IAM_AUTH from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME from onyx.configs.constants import REDIS_SOCKET_KEEPALIVE_OPTIONS +from onyx.redis.iam_auth import configure_redis_iam_auth +from onyx.redis.iam_auth import create_redis_ssl_context_if_iam from onyx.utils.logger import setup_logger from shared_configs.configs import DEFAULT_REDIS_PREFIX from shared_configs.contextvars import get_current_tenant_id @@ -186,12 +189,43 @@ def create_pool( ssl_cert_reqs: str = REDIS_SSL_CERT_REQS, ssl: bool = False, ) -> redis.BlockingConnectionPool: + """ + Create a Redis connection pool with appropriate SSL configuration. + + SSL Configuration Priority: + 1. IAM Authentication (USE_REDIS_IAM_AUTH=true): Uses system CA certificates + 2. Regular SSL (REDIS_SSL=true): Uses custom SSL configuration + 3. No SSL: Standard connection without encryption + + Note: IAM authentication automatically enables SSL and takes precedence + over regular SSL configuration to ensure proper security. + """ """We use BlockingConnectionPool because it will block and wait for a connection rather than error if max_connections is reached. This is far more deterministic behavior and aligned with how we want to use Redis.""" # Using ConnectionPool is not well documented. # Useful examples: https://github.com/redis/redis-py/issues/780 + + # Handle IAM authentication + if USE_REDIS_IAM_AUTH: + # For IAM authentication, we don't use password + # and ensure SSL is enabled with proper context + ssl_context = create_redis_ssl_context_if_iam() + return redis.BlockingConnectionPool( + host=host, + port=port, + db=db, + password=None, # No password with IAM auth + max_connections=max_connections, + timeout=None, + health_check_interval=REDIS_HEALTH_CHECK_INTERVAL, + socket_keepalive=True, + socket_keepalive_options=REDIS_SOCKET_KEEPALIVE_OPTIONS, + connection_class=redis.SSLConnection, + ssl_context=ssl_context, # Use IAM auth SSL context + ) + if ssl: return redis.BlockingConnectionPool( host=host, @@ -363,7 +397,15 @@ async def get_async_redis_connection() -> aioredis.Redis: "socket_keepalive_options": REDIS_SOCKET_KEEPALIVE_OPTIONS, } - if REDIS_SSL: + # Handle SSL configuration with clear priority: + # 1. IAM Authentication (takes precedence, handles SSL automatically) + # 2. Regular SSL (only when IAM auth is disabled) + if USE_REDIS_IAM_AUTH: + # IAM authentication handles SSL configuration automatically + # This ensures proper security with system CA certificates + configure_redis_iam_auth(connection_kwargs) + elif REDIS_SSL: + # Regular SSL configuration (only when not using IAM auth) ssl_context = ssl.create_default_context() if REDIS_SSL_CA_CERTS: