feat: parallelized integration tests #2346
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run MIT Integration Tests v2 | |
| concurrency: | |
| group: Run-MIT-Integration-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }} | |
| cancel-in-progress: true | |
| on: | |
| merge_group: | |
| pull_request: | |
| branches: | |
| - main | |
| - "release/**" | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }} | |
| CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }} | |
| CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} | |
| JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} | |
| JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} | |
| JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} | |
| PLATFORM_PAIR: linux-amd64 | |
| jobs: | |
| discover-test-dirs: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test-dirs: ${{ steps.set-matrix.outputs.test-dirs }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Discover test directories | |
| id: set-matrix | |
| run: | | |
| # Find all leaf-level directories in both test directories | |
| tests_dirs=$(find backend/tests/integration/tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort) | |
| connector_dirs=$(find backend/tests/integration/connector_job_tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort) | |
| # Create JSON array with directory info | |
| all_dirs="" | |
| for dir in $tests_dirs; do | |
| all_dirs="$all_dirs{\"path\":\"tests/$dir\",\"name\":\"tests-$dir\"}," | |
| done | |
| for dir in $connector_dirs; do | |
| all_dirs="$all_dirs{\"path\":\"connector_job_tests/$dir\",\"name\":\"connector-$dir\"}," | |
| done | |
| # Remove trailing comma and wrap in array | |
| all_dirs="[${all_dirs%,}]" | |
| echo "test-dirs=$all_dirs" >> $GITHUB_OUTPUT | |
| build-images: | |
| runs-on: | |
| [ | |
| runs-on, | |
| runner=16cpu-linux-x64, | |
| disk=large, | |
| "run-id=${{ github.run_id }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| backend/requirements/default.txt | |
| backend/requirements/dev.txt | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install --retries 5 --timeout 30 -r backend/requirements/default.txt | |
| pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt | |
| - name: Generate OpenAPI schema | |
| working-directory: ./backend | |
| env: | |
| PYTHONPATH: "." | |
| run: | | |
| python scripts/onyx_openapi_schema.py --filename generated/openapi.json | |
| - name: Generate OpenAPI Python client | |
| working-directory: ./backend | |
| run: | | |
| docker run --rm \ | |
| -v "${{ github.workspace }}/backend/generated:/local" \ | |
| openapitools/openapi-generator-cli generate \ | |
| -i /local/openapi.json \ | |
| -g python \ | |
| -o /local/onyx_openapi_client \ | |
| --package-name onyx_openapi_client | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| # Pull Web Docker image and tag for testing | |
| - name: Pull and tag Web Docker image | |
| run: | | |
| docker pull onyxdotapp/onyx-web-server:latest | |
| docker tag onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:test-${{ github.run_id }} | |
| docker push onyxdotapp/onyx-web-server:test-${{ github.run_id }} | |
| # Build and push Backend Docker image | |
| - name: Build and push Backend Docker image | |
| uses: ./.github/actions/custom-build-and-push | |
| with: | |
| context: ./backend | |
| file: ./backend/Dockerfile | |
| platforms: linux/amd64 | |
| tags: onyxdotapp/onyx-backend:test-${{ github.run_id }} | |
| push: true | |
| cache-from: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} | |
| cache-to: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max | |
| # Build and push Model Server Docker image | |
| - name: Build and push Model Server Docker image | |
| uses: ./.github/actions/custom-build-and-push | |
| with: | |
| context: ./backend | |
| file: ./backend/Dockerfile.model_server | |
| platforms: linux/amd64 | |
| tags: onyxdotapp/onyx-model-server:test-${{ github.run_id }} | |
| push: true | |
| cache-from: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} | |
| cache-to: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max | |
| # Build and push integration test Docker image | |
| - name: Build and push integration test Docker image | |
| uses: ./.github/actions/custom-build-and-push | |
| with: | |
| context: ./backend | |
| file: ./backend/tests/integration/Dockerfile | |
| platforms: linux/amd64 | |
| tags: onyxdotapp/onyx-integration:test-${{ github.run_id }} | |
| push: true | |
| cache-from: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} | |
| cache-to: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max | |
| integration-tests-mit: | |
| needs: [discover-test-dirs, build-images] | |
| # See https://runs-on.com/runners/linux/ | |
| runs-on: | |
| [ | |
| runs-on, | |
| runner=4cpu-linux-x64, | |
| disk=large, | |
| "run-id=${{ github.run_id }}", | |
| ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| # Pull pre-built images | |
| - name: Pull pre-built Docker images | |
| run: | | |
| docker pull onyxdotapp/onyx-web-server:test-${{ github.run_id }} | |
| docker pull onyxdotapp/onyx-backend:test-${{ github.run_id }} | |
| docker pull onyxdotapp/onyx-model-server:test-${{ github.run_id }} | |
| docker pull onyxdotapp/onyx-integration:test-${{ github.run_id }} | |
| # Re-tag with expected names for docker-compose | |
| docker tag onyxdotapp/onyx-web-server:test-${{ github.run_id }} onyxdotapp/onyx-web-server:test | |
| docker tag onyxdotapp/onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test | |
| docker tag onyxdotapp/onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test | |
| docker tag onyxdotapp/onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test | |
| # NOTE: Use pre-ping/null pool to reduce flakiness due to dropped connections | |
| - name: Start Docker containers | |
| run: | | |
| cd deployment/docker_compose | |
| AUTH_TYPE=basic \ | |
| POSTGRES_POOL_PRE_PING=true \ | |
| POSTGRES_USE_NULL_POOL=true \ | |
| REQUIRE_EMAIL_VERIFICATION=false \ | |
| DISABLE_TELEMETRY=true \ | |
| IMAGE_TAG=test \ | |
| INTEGRATION_TESTS_MODE=true \ | |
| docker compose -f docker-compose.dev.yml -p onyx-stack up -d | |
| id: start_docker | |
| - name: Wait for service to be ready | |
| run: | | |
| echo "Starting wait-for-service script..." | |
| docker logs -f onyx-stack-api_server-1 & | |
| start_time=$(date +%s) | |
| timeout=300 # 5 minutes in seconds | |
| while true; do | |
| current_time=$(date +%s) | |
| elapsed_time=$((current_time - start_time)) | |
| if [ $elapsed_time -ge $timeout ]; then | |
| echo "Timeout reached. Service did not become ready in 5 minutes." | |
| exit 1 | |
| fi | |
| # Use curl with error handling to ignore specific exit code 56 | |
| response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health || echo "curl_error") | |
| if [ "$response" = "200" ]; then | |
| echo "Service is ready!" | |
| break | |
| elif [ "$response" = "curl_error" ]; then | |
| echo "Curl encountered an error, possibly exit code 56. Continuing to retry..." | |
| else | |
| echo "Service not ready yet (HTTP status $response). Retrying in 5 seconds..." | |
| fi | |
| sleep 5 | |
| done | |
| echo "Finished waiting for service." | |
| - name: Start Mock Services | |
| run: | | |
| cd backend/tests/integration/mock_services | |
| docker compose -f docker-compose.mock-it-services.yml \ | |
| -p mock-it-services-stack up -d | |
| # NOTE: Use pre-ping/null to reduce flakiness due to dropped connections | |
| - name: Run Integration Tests for ${{ matrix.test-dir.name }} | |
| run: | | |
| echo "Running integration tests for ${{ matrix.test-dir.path }}..." | |
| docker run --rm --network onyx-stack_default \ | |
| --name test-runner \ | |
| -e POSTGRES_HOST=relational_db \ | |
| -e POSTGRES_USER=postgres \ | |
| -e POSTGRES_PASSWORD=password \ | |
| -e POSTGRES_DB=postgres \ | |
| -e DB_READONLY_USER=db_readonly_user \ | |
| -e DB_READONLY_PASSWORD=password \ | |
| -e POSTGRES_POOL_PRE_PING=true \ | |
| -e POSTGRES_USE_NULL_POOL=true \ | |
| -e VESPA_HOST=index \ | |
| -e REDIS_HOST=cache \ | |
| -e API_SERVER_HOST=api_server \ | |
| -e OPENAI_API_KEY=${OPENAI_API_KEY} \ | |
| -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \ | |
| -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \ | |
| -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \ | |
| -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \ | |
| -e JIRA_BASE_URL=${JIRA_BASE_URL} \ | |
| -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \ | |
| -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \ | |
| -e TEST_WEB_HOSTNAME=test-runner \ | |
| -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \ | |
| -e MOCK_CONNECTOR_SERVER_PORT=8001 \ | |
| onyxdotapp/onyx-integration:test \ | |
| /app/tests/integration/${{ matrix.test-dir.path }} | |
| continue-on-error: true | |
| id: run_tests | |
| - name: Check test results | |
| run: | | |
| if [ ${{ steps.run_tests.outcome }} == 'failure' ]; then | |
| echo "Integration tests failed. Exiting with error." | |
| exit 1 | |
| else | |
| echo "All integration tests passed successfully." | |
| fi | |
| # ------------------------------------------------------------ | |
| # Always gather logs BEFORE "down": | |
| - name: Dump API server logs | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.dev.yml -p onyx-stack logs --no-color api_server > $GITHUB_WORKSPACE/api_server.log || true | |
| - name: Dump all-container logs (optional) | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.dev.yml -p onyx-stack logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-all-logs-${{ matrix.test-dir.name }} | |
| path: ${{ github.workspace }}/docker-compose.log | |
| # ------------------------------------------------------------ | |
| - name: Stop Docker containers | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.dev.yml -p onyx-stack down -v |