From 604c2cc87611ae6dcb4655f3701d2a30cf3f3b02 Mon Sep 17 00:00:00 2001 From: Weves Date: Thu, 25 Sep 2025 13:42:49 -0700 Subject: [PATCH 1/5] feat(docker): Add DEV_MODE flag for exposing service ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add single DEV_MODE environment variable to control port exposure - When DEV_MODE=true, exposes ports for api_server, postgres, vespa, redis, and minio - Update external dependency unit test workflow to use DEV_MODE=true - Simplifies switching between production (no exposed ports) and dev/test configurations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .../pr-external-dependency-unit-tests.yml | 2 +- deployment/docker_compose/docker-compose.yml | 29 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr-external-dependency-unit-tests.yml b/.github/workflows/pr-external-dependency-unit-tests.yml index 8d740ceedc9..e6df8732c89 100644 --- a/.github/workflows/pr-external-dependency-unit-tests.yml +++ b/.github/workflows/pr-external-dependency-unit-tests.yml @@ -79,7 +79,7 @@ jobs: - name: Set up Standard Dependencies run: | cd deployment/docker_compose - docker compose up -d minio relational_db cache index + DEV_MODE=true docker compose up -d minio relational_db cache index - name: Wait for services run: | diff --git a/deployment/docker_compose/docker-compose.yml b/deployment/docker_compose/docker-compose.yml index db99a130fd3..3c621b888b7 100644 --- a/deployment/docker_compose/docker-compose.yml +++ b/deployment/docker_compose/docker-compose.yml @@ -53,8 +53,9 @@ services: - inference_model_server - minio restart: unless-stopped - # ports: - # - "8080:8080" + ports: + # DEV MODE: Exposed when DEV_MODE=true + - "${DEV_MODE:+8080:8080}" environment: # Auth Settings - AUTH_TYPE=${AUTH_TYPE:-basic} @@ -228,8 +229,9 @@ services: environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password} - # ports: - # - "5432:5432" + ports: + # DEV MODE: Exposed when DEV_MODE=true + - "${DEV_MODE:+5432:5432}" volumes: - db_volume:/var/lib/postgresql/data @@ -239,9 +241,10 @@ services: restart: unless-stopped environment: - VESPA_SKIP_UPGRADE_CHECK=${VESPA_SKIP_UPGRADE_CHECK:-true} - # ports: - # - "19071:19071" - # - "8081:8081" + ports: + # DEV MODE: Exposed when DEV_MODE=true + - "${DEV_MODE:+19071:19071}" + - "${DEV_MODE:+8081:8081}" volumes: - vespa_volume:/opt/vespa/var logging: @@ -286,8 +289,9 @@ services: cache: image: redis:7.4-alpine restart: unless-stopped - # ports: - # - "6379:6379" + ports: + # DEV MODE: Exposed when DEV_MODE=true + - "${DEV_MODE:+6379:6379}" # docker silently mounts /data even without an explicit volume mount, which enables # persistence. explicitly setting save and appendonly forces ephemeral behavior. command: redis-server --save "" --appendonly no @@ -295,9 +299,10 @@ services: minio: image: minio/minio:RELEASE.2025-07-23T15-54-02Z-cpuv1 restart: unless-stopped - # ports: - # - "9004:9000" - # - "9005:9001" + ports: + # DEV MODE: Exposed when DEV_MODE=true + - "${DEV_MODE:+9000:9000}" + - "${DEV_MODE:+9001:9001}" environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} From 22f01d50e68ea35a0d3ff511fcd30c3dfef05d49 Mon Sep 17 00:00:00 2001 From: Weves Date: Thu, 25 Sep 2025 13:57:06 -0700 Subject: [PATCH 2/5] test --- .../pr-external-dependency-unit-tests.yml | 2 +- .../docker_compose/docker-compose.dev.yml | 32 +++++++++++++++++ deployment/docker_compose/docker-compose.yml | 34 +++++++++---------- 3 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 deployment/docker_compose/docker-compose.dev.yml diff --git a/.github/workflows/pr-external-dependency-unit-tests.yml b/.github/workflows/pr-external-dependency-unit-tests.yml index e6df8732c89..74e983d6724 100644 --- a/.github/workflows/pr-external-dependency-unit-tests.yml +++ b/.github/workflows/pr-external-dependency-unit-tests.yml @@ -79,7 +79,7 @@ jobs: - name: Set up Standard Dependencies run: | cd deployment/docker_compose - DEV_MODE=true docker compose up -d minio relational_db cache index + docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d minio relational_db cache index - name: Wait for services run: | diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml new file mode 100644 index 00000000000..f868e651934 --- /dev/null +++ b/deployment/docker_compose/docker-compose.dev.yml @@ -0,0 +1,32 @@ +# Docker Compose Override for Development/Testing +# This file exposes service ports for development and testing purposes +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d +# +# Or set COMPOSE_FILE environment variable: +# export COMPOSE_FILE=docker-compose.yml:docker-compose.dev.yml +# docker compose up -d + +services: + api_server: + ports: + - "8080:8080" + + relational_db: + ports: + - "5432:5432" + + index: + ports: + - "19071:19071" + - "8081:8081" + + cache: + ports: + - "6379:6379" + + minio: + ports: + - "9000:9000" + - "9001:9001" \ No newline at end of file diff --git a/deployment/docker_compose/docker-compose.yml b/deployment/docker_compose/docker-compose.yml index 3c621b888b7..2333fa3d976 100644 --- a/deployment/docker_compose/docker-compose.yml +++ b/deployment/docker_compose/docker-compose.yml @@ -53,9 +53,9 @@ services: - inference_model_server - minio restart: unless-stopped - ports: - # DEV MODE: Exposed when DEV_MODE=true - - "${DEV_MODE:+8080:8080}" + # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # ports: + # - "8080:8080" environment: # Auth Settings - AUTH_TYPE=${AUTH_TYPE:-basic} @@ -229,9 +229,9 @@ services: environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password} - ports: - # DEV MODE: Exposed when DEV_MODE=true - - "${DEV_MODE:+5432:5432}" + # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # ports: + # - "5432:5432" volumes: - db_volume:/var/lib/postgresql/data @@ -241,10 +241,10 @@ services: restart: unless-stopped environment: - VESPA_SKIP_UPGRADE_CHECK=${VESPA_SKIP_UPGRADE_CHECK:-true} - ports: - # DEV MODE: Exposed when DEV_MODE=true - - "${DEV_MODE:+19071:19071}" - - "${DEV_MODE:+8081:8081}" + # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # ports: + # - "19071:19071" + # - "8081:8081" volumes: - vespa_volume:/opt/vespa/var logging: @@ -289,9 +289,9 @@ services: cache: image: redis:7.4-alpine restart: unless-stopped - ports: - # DEV MODE: Exposed when DEV_MODE=true - - "${DEV_MODE:+6379:6379}" + # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # ports: + # - "6379:6379" # docker silently mounts /data even without an explicit volume mount, which enables # persistence. explicitly setting save and appendonly forces ephemeral behavior. command: redis-server --save "" --appendonly no @@ -299,10 +299,10 @@ services: minio: image: minio/minio:RELEASE.2025-07-23T15-54-02Z-cpuv1 restart: unless-stopped - ports: - # DEV MODE: Exposed when DEV_MODE=true - - "${DEV_MODE:+9000:9000}" - - "${DEV_MODE:+9001:9001}" + # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # ports: + # - "9000:9000" + # - "9001:9001" environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} From c1802078d2d4f754f57548801a3a30e1adea912a Mon Sep 17 00:00:00 2001 From: Weves Date: Thu, 25 Sep 2025 14:15:46 -0700 Subject: [PATCH 3/5] fix --- deployment/docker_compose/docker-compose.dev.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml index f868e651934..16fec217632 100644 --- a/deployment/docker_compose/docker-compose.dev.yml +++ b/deployment/docker_compose/docker-compose.dev.yml @@ -27,6 +27,7 @@ services: - "6379:6379" minio: + # use different ports to avoid conflicts with model servers ports: - - "9000:9000" - - "9001:9001" \ No newline at end of file + - "9004:9000" + - "9005:9001" \ No newline at end of file From 95ab8227dbcf52d54564d74200766780e5c04c01 Mon Sep 17 00:00:00 2001 From: Weves Date: Thu, 25 Sep 2025 14:20:28 -0700 Subject: [PATCH 4/5] polish --- deployment/docker_compose/docker-compose.yml | 24 ++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/deployment/docker_compose/docker-compose.yml b/deployment/docker_compose/docker-compose.yml index 2333fa3d976..0044c3cb881 100644 --- a/deployment/docker_compose/docker-compose.yml +++ b/deployment/docker_compose/docker-compose.yml @@ -53,7 +53,9 @@ services: - inference_model_server - minio restart: unless-stopped - # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # DEV: To expose ports, either: + # 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + # 2. Uncomment the ports below # ports: # - "8080:8080" environment: @@ -229,7 +231,9 @@ services: environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password} - # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # DEV: To expose ports, either: + # 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + # 2. Uncomment the ports below # ports: # - "5432:5432" volumes: @@ -241,7 +245,9 @@ services: restart: unless-stopped environment: - VESPA_SKIP_UPGRADE_CHECK=${VESPA_SKIP_UPGRADE_CHECK:-true} - # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # DEV: To expose ports, either: + # 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + # 2. Uncomment the ports below # ports: # - "19071:19071" # - "8081:8081" @@ -289,7 +295,9 @@ services: cache: image: redis:7.4-alpine restart: unless-stopped - # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # DEV: To expose ports, either: + # 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + # 2. Uncomment the ports below # ports: # - "6379:6379" # docker silently mounts /data even without an explicit volume mount, which enables @@ -299,10 +307,12 @@ services: minio: image: minio/minio:RELEASE.2025-07-23T15-54-02Z-cpuv1 restart: unless-stopped - # DEV MODE: Uncomment or set DEV_MODE=true to expose ports + # DEV: To expose ports, either: + # 1. Use docker-compose.dev.yml: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + # 2. Uncomment the ports below # ports: - # - "9000:9000" - # - "9001:9001" + # - "9004:9000" + # - "9005:9001" environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} From 6d00b13779f7c816d94d25987a920441b3e230b4 Mon Sep 17 00:00:00 2001 From: justin-tahara Date: Thu, 25 Sep 2025 14:47:47 -0700 Subject: [PATCH 5/5] Adding same fix for the intergration tests to use the dev docker compose file --- .github/workflows/pr-integration-tests.yml | 2 +- .github/workflows/pr-mit-integration-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-integration-tests.yml b/.github/workflows/pr-integration-tests.yml index dcf25ff3fb1..5737958a90a 100644 --- a/.github/workflows/pr-integration-tests.yml +++ b/.github/workflows/pr-integration-tests.yml @@ -262,7 +262,7 @@ jobs: IMAGE_TAG=test \ INTEGRATION_TESTS_MODE=true \ CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS=0.001 \ - docker compose up \ + docker compose -f docker-compose.yml -f docker-compose.dev.yml up \ relational_db \ index \ cache \ diff --git a/.github/workflows/pr-mit-integration-tests.yml b/.github/workflows/pr-mit-integration-tests.yml index 15806e7fe2e..b20a74bac6b 100644 --- a/.github/workflows/pr-mit-integration-tests.yml +++ b/.github/workflows/pr-mit-integration-tests.yml @@ -258,7 +258,7 @@ jobs: DISABLE_TELEMETRY=true \ IMAGE_TAG=test \ INTEGRATION_TESTS_MODE=true \ - docker compose up \ + docker compose -f docker-compose.yml -f docker-compose.dev.yml up \ relational_db \ index \ cache \