From d8e2417ff116dcc8066a788fc8b8e2bcbea8ef22 Mon Sep 17 00:00:00 2001 From: Wenxi Onyx Date: Tue, 26 Aug 2025 16:12:28 -0700 Subject: [PATCH 1/4] add api/versions to onyx --- backend/onyx/server/auth_check.py | 2 + backend/onyx/server/manage/get_state.py | 110 ++++++++++++++++++++++++ backend/onyx/server/manage/models.py | 13 +++ 3 files changed, 125 insertions(+) diff --git a/backend/onyx/server/auth_check.py b/backend/onyx/server/auth_check.py index bb7a1518989..7581acc8ffd 100644 --- a/backend/onyx/server/auth_check.py +++ b/backend/onyx/server/auth_check.py @@ -30,6 +30,8 @@ ("/auth/type", {"GET"}), # just gets the version of Onyx (e.g. 0.3.11) ("/version", {"GET"}), + # Gets stable and beta versions for Onyx docker images + ("/versions", {"GET"}), # stuff related to basic auth ("/auth/refresh", {"POST"}), ("/auth/register", {"POST"}), diff --git a/backend/onyx/server/manage/get_state.py b/backend/onyx/server/manage/get_state.py index 48ea45c5af1..a6b04e585bf 100644 --- a/backend/onyx/server/manage/get_state.py +++ b/backend/onyx/server/manage/get_state.py @@ -1,10 +1,17 @@ +import concurrent.futures +import re + +import requests from fastapi import APIRouter +from fastapi import HTTPException from onyx import __version__ from onyx.auth.users import anonymous_user_enabled from onyx.auth.users import user_needs_to_be_verified from onyx.configs.app_configs import AUTH_TYPE +from onyx.server.manage.models import AllVersions from onyx.server.manage.models import AuthTypeResponse +from onyx.server.manage.models import ContainerVersions from onyx.server.manage.models import VersionResponse from onyx.server.models import StatusResponse @@ -28,3 +35,106 @@ def get_auth_type() -> AuthTypeResponse: @router.get("/version") def get_version() -> VersionResponse: return VersionResponse(backend_version=__version__) + + +@router.get("/versions") +def get_versions() -> AllVersions: + """ + Fetches the latest stable and beta versions of Onyx Docker images + """ + # Define strict version patterns + STABLE_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") + DEV_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$") + + # Fetch the latest tags from DockerHub for each Onyx component + dockerhub_repos = [ + "onyxdotapp/onyx-model-server", + "onyxdotapp/onyx-backend", + "onyxdotapp/onyx-web-server", + ] + + # For good measure, we fetch 10 pages of tags + def get_dockerhub_tags(repo: str, pages: int = 10) -> list[str]: + url = f"https://hub.docker.com/v2/repositories/{repo}/tags" + tags = [] + for _ in range(pages): + response = requests.get(url) + response.raise_for_status() + data = response.json() + tags.extend( + [ + tag["name"] + for tag in data["results"] + if re.match(r"^v\d", tag["name"]) + ] + ) + url = data.get("next") + if not url: + break + return tags + + # Get tags for all repos in parallel + with concurrent.futures.ThreadPoolExecutor() as executor: + all_tags = list( + executor.map(lambda repo: set(get_dockerhub_tags(repo)), dockerhub_repos) + ) + + # Find common tags across all repos + common_tags = set.intersection(*all_tags) + + # Filter tags by strict version patterns + dev_tags = [tag for tag in common_tags if DEV_VERSION_PATTERN.match(tag)] + stable_tags = [tag for tag in common_tags if STABLE_VERSION_PATTERN.match(tag)] + + # Ensure we have at least one tag of each type + if not dev_tags: + raise HTTPException( + status_code=500, + detail="No valid dev versions found matching pattern v(number).(number).(number)-beta", + ) + if not stable_tags: + raise HTTPException( + status_code=500, + detail="No valid stable versions found matching pattern v(number).(number).(number)", + ) + + # Sort common tags and get the latest one + def version_key(version: str) -> tuple[int, int, int, int]: + """Extract major, minor, patch, beta as integers for sorting""" + # Remove 'v' prefix + clean_version = version[1:] + + # Check if it's a beta version + if "-beta." in clean_version: + # Split on '-beta.' to separate version and beta number + base_version, beta_num = clean_version.split("-beta.") + parts = base_version.split(".") + return (int(parts[0]), int(parts[1]), int(parts[2]), int(beta_num)) + else: + # Stable version - no beta number + parts = clean_version.split(".") + return (int(parts[0]), int(parts[1]), int(parts[2]), 0) + + latest_dev_version = sorted(dev_tags, key=version_key, reverse=True)[0] + latest_stable_version = sorted(stable_tags, key=version_key, reverse=True)[0] + + return AllVersions( + stable=ContainerVersions( + danswer=latest_stable_version, + relational_db="postgres:15.2-alpine", + index="vespaengine/vespa:8.277.17", + nginx="nginx:1.23.4-alpine", + ), + dev=ContainerVersions( + danswer=latest_dev_version, + relational_db="postgres:15.2-alpine", + index="vespaengine/vespa:8.277.17", + nginx="nginx:1.23.4-alpine", + ), + migration=ContainerVersions( + danswer="airgapped-intfloat-nomic-migration", + relational_db="postgres:15.2-alpine", + index="vespaengine/vespa:8.277.17", + nginx="nginx:1.23.4-alpine", + ), + ) diff --git a/backend/onyx/server/manage/models.py b/backend/onyx/server/manage/models.py index e7091d18dc1..7122ab27960 100644 --- a/backend/onyx/server/manage/models.py +++ b/backend/onyx/server/manage/models.py @@ -406,3 +406,16 @@ def validate_keyword_if_regex(self) -> Any: ["invalid regex pattern", pattern, f"in `keyword`: {err.msg}"] ) ) + + +class ContainerVersions(BaseModel): + danswer: str + relational_db: str + index: str + nginx: str + + +class AllVersions(BaseModel): + stable: ContainerVersions + dev: ContainerVersions + migration: ContainerVersions From 08a515b68f571a726be1544ab748a3e2842b7c59 Mon Sep 17 00:00:00 2001 From: Wenxi Onyx Date: Tue, 26 Aug 2025 16:27:31 -0700 Subject: [PATCH 2/4] add test and rename onyx --- backend/onyx/configs/app_configs.py | 8 +++ backend/onyx/server/manage/get_state.py | 14 +++-- backend/onyx/server/manage/models.py | 2 +- backend/tests/api/test_api.py | 70 +++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 9 deletions(-) diff --git a/backend/onyx/configs/app_configs.py b/backend/onyx/configs/app_configs.py index cbc33d1f6c8..adf1b40b787 100644 --- a/backend/onyx/configs/app_configs.py +++ b/backend/onyx/configs/app_configs.py @@ -1,5 +1,6 @@ import json import os +import re import urllib.parse from datetime import datetime from datetime import timezone @@ -26,6 +27,13 @@ SKIP_WARM_UP = os.environ.get("SKIP_WARM_UP", "").lower() == "true" +##### +# Version Pattern Configs +##### +# Version patterns for Docker image tags +STABLE_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") +DEV_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$") + ##### # User Facing Features Configs ##### diff --git a/backend/onyx/server/manage/get_state.py b/backend/onyx/server/manage/get_state.py index a6b04e585bf..a265afedde5 100644 --- a/backend/onyx/server/manage/get_state.py +++ b/backend/onyx/server/manage/get_state.py @@ -9,6 +9,8 @@ from onyx.auth.users import anonymous_user_enabled from onyx.auth.users import user_needs_to_be_verified from onyx.configs.app_configs import AUTH_TYPE +from onyx.configs.app_configs import DEV_VERSION_PATTERN +from onyx.configs.app_configs import STABLE_VERSION_PATTERN from onyx.server.manage.models import AllVersions from onyx.server.manage.models import AuthTypeResponse from onyx.server.manage.models import ContainerVersions @@ -42,10 +44,6 @@ def get_versions() -> AllVersions: """ Fetches the latest stable and beta versions of Onyx Docker images """ - # Define strict version patterns - STABLE_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") - DEV_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$") - # Fetch the latest tags from DockerHub for each Onyx component dockerhub_repos = [ "onyxdotapp/onyx-model-server", @@ -90,7 +88,7 @@ def get_dockerhub_tags(repo: str, pages: int = 10) -> list[str]: if not dev_tags: raise HTTPException( status_code=500, - detail="No valid dev versions found matching pattern v(number).(number).(number)-beta", + detail="No valid dev versions found matching pattern v(number).(number).(number)-beta.(number)", ) if not stable_tags: raise HTTPException( @@ -120,19 +118,19 @@ def version_key(version: str) -> tuple[int, int, int, int]: return AllVersions( stable=ContainerVersions( - danswer=latest_stable_version, + onyx=latest_stable_version, relational_db="postgres:15.2-alpine", index="vespaengine/vespa:8.277.17", nginx="nginx:1.23.4-alpine", ), dev=ContainerVersions( - danswer=latest_dev_version, + onyx=latest_dev_version, relational_db="postgres:15.2-alpine", index="vespaengine/vespa:8.277.17", nginx="nginx:1.23.4-alpine", ), migration=ContainerVersions( - danswer="airgapped-intfloat-nomic-migration", + onyx="airgapped-intfloat-nomic-migration", relational_db="postgres:15.2-alpine", index="vespaengine/vespa:8.277.17", nginx="nginx:1.23.4-alpine", diff --git a/backend/onyx/server/manage/models.py b/backend/onyx/server/manage/models.py index 7122ab27960..56dca1ff72a 100644 --- a/backend/onyx/server/manage/models.py +++ b/backend/onyx/server/manage/models.py @@ -409,7 +409,7 @@ def validate_keyword_if_regex(self) -> Any: class ContainerVersions(BaseModel): - danswer: str + onyx: str relational_db: str index: str nginx: str diff --git a/backend/tests/api/test_api.py b/backend/tests/api/test_api.py index 45e5cebc7d7..c7c8f88088c 100644 --- a/backend/tests/api/test_api.py +++ b/backend/tests/api/test_api.py @@ -6,6 +6,8 @@ from fastapi import FastAPI from fastapi.testclient import TestClient +from onyx.configs.app_configs import DEV_VERSION_PATTERN +from onyx.configs.app_configs import STABLE_VERSION_PATTERN from onyx.main import fetch_versioned_implementation from onyx.utils.logger import setup_logger @@ -103,3 +105,71 @@ def test_handle_send_message_simple_with_history(client: TestClient) -> None: # persona must have LLM relevance enabled for this to pass assert len(resp_json["llm_selected_doc_indices"]) > 0 + + +def test_versions_endpoint(client: TestClient) -> None: + """Test that /api/versions endpoint returns valid stable, dev, and migration configurations""" + response = client.get("/versions") + assert response.status_code == 200 + + data = response.json() + + # Verify the top-level structure + assert "stable" in data + assert "dev" in data + assert "migration" in data + + # Verify stable configuration + stable = data["stable"] + assert "onyx" in stable + assert "relational_db" in stable + assert "index" in stable + assert "nginx" in stable + + # Verify stable version follows correct pattern (v1.2.3) + # If this fails, revise latest Github release for typo or incorrect version name + assert STABLE_VERSION_PATTERN.match( + stable["onyx"] + ), f"Stable version {stable['onyx']} doesn't match pattern v(number).(number).(number)" + + # Verify dev configuration + dev = data["dev"] + assert "onyx" in dev + assert "relational_db" in dev + assert "index" in dev + assert "nginx" in dev + + # Verify dev version follows correct pattern (v1.2.3-beta.4) + assert DEV_VERSION_PATTERN.match( + dev["onyx"] + ), f"Dev version {dev['onyx']} doesn't match pattern v(number).(number).(number)-beta.(number)" + + # Verify migration configuration + migration = data["migration"] + assert "onyx" in migration + assert "relational_db" in migration + assert "index" in migration + assert "nginx" in migration + + # Verify migration has expected values + assert migration["onyx"] == "airgapped-intfloat-nomic-migration" + assert migration["relational_db"] == "postgres:15.2-alpine" + assert migration["index"] == "vespaengine/vespa:8.277.17" + assert migration["nginx"] == "nginx:1.23.4-alpine" + + # Verify versions are different between stable and dev + assert stable["onyx"] != dev["onyx"], "Stable and dev versions should be different" + + # Additional validation: ensure all required fields are strings + for config_name, config in [ + ("stable", stable), + ("dev", dev), + ("migration", migration), + ]: + for field_name, field_value in config.items(): + assert isinstance( + field_value, str + ), f"{config_name}.{field_name} should be a string, got {type(field_value)}" + assert ( + field_value.strip() != "" + ), f"{config_name}.{field_name} should not be empty" From 2817da6e3ea130ee8af63aa9c6b1ba5a6a26a982 Mon Sep 17 00:00:00 2001 From: Wenxi Date: Tue, 26 Aug 2025 16:28:37 -0700 Subject: [PATCH 3/4] cubic nit Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- backend/onyx/server/manage/get_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/onyx/server/manage/get_state.py b/backend/onyx/server/manage/get_state.py index a265afedde5..a1df267d1f4 100644 --- a/backend/onyx/server/manage/get_state.py +++ b/backend/onyx/server/manage/get_state.py @@ -56,7 +56,7 @@ def get_dockerhub_tags(repo: str, pages: int = 10) -> list[str]: url = f"https://hub.docker.com/v2/repositories/{repo}/tags" tags = [] for _ in range(pages): - response = requests.get(url) + response = requests.get(url, timeout=10) response.raise_for_status() data = response.json() tags.extend( From 9db9dd46ddcb6a6daa5ca4aa37b96f90d3a80d14 Mon Sep 17 00:00:00 2001 From: Wenxi Onyx Date: Tue, 26 Aug 2025 17:44:47 -0700 Subject: [PATCH 4/4] move api version constants and add explanatory comment --- backend/onyx/configs/app_configs.py | 8 -------- backend/onyx/configs/constants.py | 7 +++++++ backend/onyx/server/manage/get_state.py | 8 +++++--- backend/tests/api/test_api.py | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/backend/onyx/configs/app_configs.py b/backend/onyx/configs/app_configs.py index adf1b40b787..cbc33d1f6c8 100644 --- a/backend/onyx/configs/app_configs.py +++ b/backend/onyx/configs/app_configs.py @@ -1,6 +1,5 @@ import json import os -import re import urllib.parse from datetime import datetime from datetime import timezone @@ -27,13 +26,6 @@ SKIP_WARM_UP = os.environ.get("SKIP_WARM_UP", "").lower() == "true" -##### -# Version Pattern Configs -##### -# Version patterns for Docker image tags -STABLE_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") -DEV_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$") - ##### # User Facing Features Configs ##### diff --git a/backend/onyx/configs/constants.py b/backend/onyx/configs/constants.py index 0cabde5b558..a775d23488b 100644 --- a/backend/onyx/configs/constants.py +++ b/backend/onyx/configs/constants.py @@ -1,4 +1,5 @@ import platform +import re import socket from enum import auto from enum import Enum @@ -46,6 +47,12 @@ "You can still use Onyx as a search engine." ) +##### +# Version Pattern Configs +##### +# Version patterns for Docker image tags +STABLE_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") +DEV_VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-beta\.(\d+)$") DEFAULT_PERSONA_ID = 0 diff --git a/backend/onyx/server/manage/get_state.py b/backend/onyx/server/manage/get_state.py index a1df267d1f4..ccb301f7559 100644 --- a/backend/onyx/server/manage/get_state.py +++ b/backend/onyx/server/manage/get_state.py @@ -9,8 +9,8 @@ from onyx.auth.users import anonymous_user_enabled from onyx.auth.users import user_needs_to_be_verified from onyx.configs.app_configs import AUTH_TYPE -from onyx.configs.app_configs import DEV_VERSION_PATTERN -from onyx.configs.app_configs import STABLE_VERSION_PATTERN +from onyx.configs.constants import DEV_VERSION_PATTERN +from onyx.configs.constants import STABLE_VERSION_PATTERN from onyx.server.manage.models import AllVersions from onyx.server.manage.models import AuthTypeResponse from onyx.server.manage.models import ContainerVersions @@ -42,7 +42,9 @@ def get_version() -> VersionResponse: @router.get("/versions") def get_versions() -> AllVersions: """ - Fetches the latest stable and beta versions of Onyx Docker images + Fetches the latest stable and beta versions of Onyx Docker images. + Since DockerHub does not explicitly flag stable and beta images, + this endpoint can be used to programmatically check for new images. """ # Fetch the latest tags from DockerHub for each Onyx component dockerhub_repos = [ diff --git a/backend/tests/api/test_api.py b/backend/tests/api/test_api.py index c7c8f88088c..a8705e7f2b8 100644 --- a/backend/tests/api/test_api.py +++ b/backend/tests/api/test_api.py @@ -6,8 +6,8 @@ from fastapi import FastAPI from fastapi.testclient import TestClient -from onyx.configs.app_configs import DEV_VERSION_PATTERN -from onyx.configs.app_configs import STABLE_VERSION_PATTERN +from onyx.configs.constants import DEV_VERSION_PATTERN +from onyx.configs.constants import STABLE_VERSION_PATTERN from onyx.main import fetch_versioned_implementation from onyx.utils.logger import setup_logger