fix(infra): fix some dependency hells and add some lazy loading to reduce celery worker RAM usage #1872
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: External Dependency Unit Tests | |
| on: | |
| merge_group: | |
| pull_request: | |
| branches: [main] | |
| env: | |
| # AWS | |
| S3_AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} | |
| S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} | |
| # MinIO | |
| S3_ENDPOINT_URL: "http://localhost:9004" | |
| # Confluence | |
| CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }} | |
| CONFLUENCE_TEST_SPACE: ${{ secrets.CONFLUENCE_TEST_SPACE }} | |
| CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }} | |
| CONFLUENCE_IS_CLOUD: ${{ secrets.CONFLUENCE_IS_CLOUD }} | |
| CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }} | |
| CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} | |
| # LLMs | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| jobs: | |
| discover-test-dirs: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test-dirs: ${{ steps.set-matrix.outputs.test-dirs }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Discover test directories | |
| id: set-matrix | |
| run: | | |
| # Find all subdirectories in backend/tests/external_dependency_unit | |
| dirs=$(find backend/tests/external_dependency_unit -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort | jq -R -s -c 'split("\n")[:-1]') | |
| echo "test-dirs=$dirs" >> $GITHUB_OUTPUT | |
| external-dependency-unit-tests: | |
| needs: discover-test-dirs | |
| # Use larger runner with more resources for Vespa | |
| runs-on: [runs-on, runner=16cpu-linux-x64, "run-id=${{ github.run_id }}"] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }} | |
| env: | |
| PYTHONPATH: ./backend | |
| MODEL_SERVER_HOST: "disabled" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| backend/requirements/default.txt | |
| backend/requirements/dev.txt | |
| - name: Install Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install --retries 5 --timeout 30 -r backend/requirements/default.txt | |
| pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt | |
| playwright install chromium | |
| playwright install-deps chromium | |
| - name: Set up Standard Dependencies | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose up -d minio relational_db cache index | |
| - name: Wait for services | |
| run: | | |
| echo "Waiting for services to be ready..." | |
| sleep 30 | |
| # Wait for Vespa specifically | |
| echo "Waiting for Vespa to be ready..." | |
| timeout 300 bash -c 'until curl -f -s http://localhost:8081/ApplicationStatus > /dev/null 2>&1; do echo "Vespa not ready, waiting..."; sleep 10; done' || echo "Vespa timeout - continuing anyway" | |
| echo "Services should be ready now" | |
| - name: Run migrations | |
| run: | | |
| cd backend | |
| # Run migrations to head | |
| alembic upgrade head | |
| alembic heads --verbose | |
| - name: Run Tests for ${{ matrix.test-dir }} | |
| shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}" | |
| run: | | |
| py.test \ | |
| --durations=8 \ | |
| -o junit_family=xunit2 \ | |
| -xv \ | |
| --ff \ | |
| backend/tests/external_dependency_unit/${{ matrix.test-dir }} |