feature: rebuild fact tables. #148
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Run on Pull Request | |
| env: | |
| FAST_WIF_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| CICD_SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} | |
| DBT_SERVICE_ACCOUNT: ${{ secrets.DBT_SERVICE_ACCOUNT_EMAIL }} | |
| TARGET: pr | |
| on: | |
| pull_request_target: | |
| types: | |
| - opened | |
| - synchronize | |
| - reopened | |
| workflow_dispatch: | |
| permissions: | |
| contents: "read" | |
| id-token: "write" | |
| pull-requests: "write" | |
| jobs: | |
| ci_job: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| environment: prod | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| - name: Set up Python | |
| run: uv python install | |
| - name: Install the project | |
| run: uv sync --dev | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ env.FAST_WIF_PROVIDER }} | |
| service_account: ${{ env.CICD_SERVICE_ACCOUNT }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Get Schema ID | |
| id: schema_id | |
| run: echo "SCHEMA_ID=${{ github.event.pull_request.number }}__${{ github.sha }}" >> $GITHUB_ENV | |
| - name: Run dbt debug | |
| run: | | |
| uv run dbt debug --target ${{ env.TARGET }} --vars "schema_id: $SCHEMA_ID" | |
| - name: Copy manifest.json from Google Cloud Storage | |
| id: check-manifest | |
| continue-on-error: true | |
| run: | | |
| mkdir -p target-base | |
| gcloud storage cp gs://${{ secrets.GCP_GCS_BUCKET }}/manifest/manifest.json target-base/ | |
| - name: Run dbt deps | |
| run: | | |
| uv run dbt deps --target ${{ env.TARGET }} --vars "schema_id: $SCHEMA_ID" | |
| - name: pre-commit | |
| uses: pre-commit/action@v3.0.1 | |
| - name: Set full refresh flag | |
| id: set-full-refresh-flag | |
| run: | | |
| if echo "${{ github.event.pull_request.body }}" | grep -q "#full-refresh"; then | |
| echo "DBT_FULL_REFRESH=--full-refresh" >> $GITHUB_OUTPUT | |
| else | |
| echo "DBT_FULL_REFRESH=" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Run dbt build | |
| id: first_try | |
| run: | | |
| uv run dbt run-operation stage_external_sources | |
| uv run dbt build --select state:modified+ --target ${{ env.TARGET }} --vars "schema_id: $SCHEMA_ID" --defer --state target-base ${{ steps.set-full-refresh-flag.outputs.DBT_FULL_REFRESH }} --indirect-selection cautious | |
| - name: Run dbt build another time to check incremental models | |
| id: incremental_try | |
| run: | | |
| uv run dbt build --select config.materialized:incremental --target ${{ env.TARGET }} --vars "schema_id: $SCHEMA_ID" --defer --state target-base |