From 8ceb2419a6345a6db8b43e0a6a07ecefc556b276 Mon Sep 17 00:00:00 2001 From: anlowee Date: Fri, 5 Sep 2025 15:27:56 +0000 Subject: [PATCH 1/3] Test --- .../presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp index fcb6d52c865bc..68d6fa2c7ab92 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp @@ -18,7 +18,7 @@ using namespace facebook::presto; using namespace facebook::velox; -class PrestoToVeloxConnectorTest : public ::testing::Test {}; +class PrestoToVeloxConnectorTest : public ::testing::Test {}; TEST_F(PrestoToVeloxConnectorTest, registerVariousConnectors) { std::vector>> From 8d37ac175f2d98621d7448af5471a11dbb203ace Mon Sep 17 00:00:00 2001 From: anlowee Date: Fri, 5 Sep 2025 19:04:06 +0000 Subject: [PATCH 2/3] Separate dep and runtime images --- .../prestissimo-worker-dep-image-build.yml | 88 ++++++++++++++++++ .../prestissimo-worker-image-build.yml | 92 +++++++++++++++++++ .../prestissimo-worker-images-build.yml | 70 -------------- 3 files changed, 180 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/prestissimo-worker-dep-image-build.yml create mode 100644 .github/workflows/prestissimo-worker-image-build.yml delete mode 100644 .github/workflows/prestissimo-worker-images-build.yml diff --git a/.github/workflows/prestissimo-worker-dep-image-build.yml b/.github/workflows/prestissimo-worker-dep-image-build.yml new file mode 100644 index 0000000000000..6684130e64687 --- /dev/null +++ b/.github/workflows/prestissimo-worker-dep-image-build.yml @@ -0,0 +1,88 @@ +name: "prestissimo-worker-dep-image-build" + +on: + pull_request: + push: + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-24.04" + outputs: + prestissimo_dependency_changed: "${{steps.filter.outputs.prestissimo_dependency}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "release-0.293-clp-connector" + filters: | + prestissimo_dependency: + - "presto-native-execution/scripts/**/*" + - "presto-native-execution/velox/scripts/**/*" + + prestissimo-worker-dep-image-build: + name: "prestissimo-worker-dep-image-build" + runs-on: "ubuntu-22.04" + env: + SHOULD_PUSH: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector' }} + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Login to image registry" + uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Set up Docker Buildx" + uses: "docker/setup-buildx-action@v3" + + - name: "Set up metadata for dependency image" + id: "metadata-deps-image" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: "ghcr.io/${{github.repository}}/prestissimo-worker-dev-env" + tags: "type=raw,value=dev" + + - name: "Build and push dependency image" + if: "'true' == needs.filter-relevant-changes.outputs.prestissimo_dependency_changed" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "./presto-native-execution" + file: "./presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile" + push: ${{ 'true' == env.SHOULD_PUSH }} + tags: "${{steps.metadata-deps-image.outputs.tags}}" + labels: "${{steps.metadata-deps-image.outputs.labels}}" + + - name: "Pack dependency image as tar" + run: | + if [[ 'true' == "${{ needs.filter-relevant-changes.outputs.prestissimo_dependency_changed }}" ]]; then + SRC_IMAGE="${{ steps.metadata-deps-image.outputs.tags }}" + else + SRC_IMAGE="ghcr.io/y-scope/presto/prestissimo-worker-dev-env:dev" + docker pull "$SRC_IMAGE" + fi + + TAR="prestissimo-worker-dev-env-image-${{ github.sha }}.tar" + docker save "$SRC_IMAGE" -o "$TAR" + echo "Saved $SRC_IMAGE to $TAR" + + - name: "Upload dependency image artifact" + uses: "actions/upload-artifact@v4" + with: + name: prestissimo-worker-dev-env-image-${{ github.sha }} + path: prestissimo-worker-dev-env-image-${{ github.sha }}.tar + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/prestissimo-worker-image-build.yml b/.github/workflows/prestissimo-worker-image-build.yml new file mode 100644 index 0000000000000..eff622a372082 --- /dev/null +++ b/.github/workflows/prestissimo-worker-image-build.yml @@ -0,0 +1,92 @@ +name: "prestissimo-worker-image-build" + +on: + workflow_run: + workflows: [ "prestissimo-worker-dep-image-build" ] # name of the producer workflow + types: [ completed ] + pull_request: + push: + +jobs: + prestissimo-worker-image-build: + name: "prestissimo-worker-image-build" + runs-on: "ubuntu-22.04" + env: + SHOULD_PUSH: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector' }} + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Login to image registry" + uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Set up Docker Buildx" + uses: "docker/setup-buildx-action@v3" + + - name: "Download dependency image artifact" + uses: "actions/download-artifact@v4" + with: + name: "prestissimo-worker-dev-env-image-${{github.event.workflow_run.head_sha}}" + path: ./_deps_image + run-id: "${{github.event.workflow_run.id}}" + github-token: "${{secrets.GITHUB_TOKEN}}" + + - name: "Load dependency image into Docker" + id: "load-deps" + shell: bash + run: | + set -euo pipefail + TAR="./_deps_image/prestissimo-worker-dev-env-image-${{github.event.workflow_run.head_sha}}.tar" + OUT="$(docker load -i "$TAR")" + echo "$OUT" + IMAGE_REF="$(echo "$OUT" | awk -F': ' '/^Loaded image:/ {print $2}' | tail -n1)" + + # Fallback if the tar had no tag and you only get an image ID: + if [[ -z "${IMAGE_REF:-}" ]]; then + ID="$(echo "$OUT" | sed -n 's/^Loaded image ID: //p' | tail -n1)" + IMAGE_REF="$(docker image inspect --format='{{index .RepoTags 0}}' "$ID" 2>/dev/null || true)" + fi + + if [[ -z "${IMAGE_REF:-}" ]]; then + echo "Could not determine image tag from docker load output" >&2 + exit 1 + fi + + echo "image_ref=$IMAGE_REF" >> "$GITHUB_OUTPUT" + echo "Loaded ${IMAGE_REF} image." + + - name: "Set up metadata for runtime image" + id: "metadata-runtime-image" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: "ghcr.io/${{github.repository}}/prestissimo-worker" + tags: "type=raw,value=dev" + + - name: "Get number of cores" + id: "get-cores" + run: |- + echo "num_cores=$(nproc)" >> $GITHUB_OUTPUT + + - name: "Build and push runtime image" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + build-args: |- + BASE_IMAGE=ubuntu:22.04 + DEPENDENCY_IMAGE=${{steps.load-deps.outputs.image_ref}} + EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_S3=ON + NUM_THREADS=${{steps.get-cores.outputs.num_cores}} + OSNAME=ubuntu + context: "./presto-native-execution" + file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile" + push: >- + ${{github.event_name != 'pull_request' + && github.ref == 'refs/heads/release-0.293-clp-connector'}} + tags: "${{steps.metadata-runtime-image.outputs.tags}}" + labels: "${{steps.metadata-runtime-image.outputs.labels}}" diff --git a/.github/workflows/prestissimo-worker-images-build.yml b/.github/workflows/prestissimo-worker-images-build.yml deleted file mode 100644 index b36dcb71949be..0000000000000 --- a/.github/workflows/prestissimo-worker-images-build.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: "prestissimo-worker-images-build" - -on: - pull_request: - push: - -jobs: - prestissimo-worker-images-build: - name: "prestissimo-worker-images-build" - runs-on: "ubuntu-22.04" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Login to image registry" - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Set up metadata for dependency image" - id: "metadata-deps-image" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: "ghcr.io/${{github.repository}}/prestissimo-worker-dev-env" - tags: "type=raw,value=dev" - - - name: "Build and push dependency image" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "./presto-native-execution" - file: "./presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile" - push: >- - ${{github.event_name != 'pull_request' - && github.ref == 'refs/heads/release-0.293-clp-connector'}} - tags: "${{steps.metadata-deps-image.outputs.tags}}" - labels: "${{steps.metadata-deps-image.outputs.labels}}" - - - name: "Set up metadata for runtime image" - id: "metadata-runtime-image" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: "ghcr.io/${{github.repository}}/prestissimo-worker" - tags: "type=raw,value=dev" - - - name: "Get number of cores" - id: "get-cores" - run: |- - echo "num_cores=$(nproc)" >> $GITHUB_OUTPUT - - - name: "Build and push runtime image" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - build-args: |- - BASE_IMAGE=ubuntu:22.04 - DEPENDENCY_IMAGE=${{steps.metadata-deps-image.outputs.tags}} - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF \ - -DPRESTO_ENABLE_PARQUET=ON \ - -DPRESTO_ENABLE_S3=ON - NUM_THREADS=${{steps.get-cores.outputs.num_cores}} - OSNAME=ubuntu - context: "./presto-native-execution" - file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile" - push: >- - ${{github.event_name != 'pull_request' - && github.ref == 'refs/heads/release-0.293-clp-connector'}} - tags: "${{steps.metadata-runtime-image.outputs.tags}}" - labels: "${{steps.metadata-runtime-image.outputs.labels}}" From 4ccb37963b138568a6d55df692a5156816ef5816 Mon Sep 17 00:00:00 2001 From: anlowee Date: Fri, 5 Sep 2025 19:28:33 +0000 Subject: [PATCH 3/3] Fix --- .github/workflows/prestissimo-worker-dep-image-build.yml | 6 +++--- .github/workflows/prestissimo-worker-image-build.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/prestissimo-worker-dep-image-build.yml b/.github/workflows/prestissimo-worker-dep-image-build.yml index 6684130e64687..884e98d1be67a 100644 --- a/.github/workflows/prestissimo-worker-dep-image-build.yml +++ b/.github/workflows/prestissimo-worker-dep-image-build.yml @@ -75,14 +75,14 @@ jobs: docker pull "$SRC_IMAGE" fi - TAR="prestissimo-worker-dev-env-image-${{ github.sha }}.tar" + TAR="prestissimo-worker-dev-env-image.tar" docker save "$SRC_IMAGE" -o "$TAR" echo "Saved $SRC_IMAGE to $TAR" - name: "Upload dependency image artifact" uses: "actions/upload-artifact@v4" with: - name: prestissimo-worker-dev-env-image-${{ github.sha }} - path: prestissimo-worker-dev-env-image-${{ github.sha }}.tar + name: prestissimo-worker-dev-env-image + path: prestissimo-worker-dev-env-image.tar if-no-files-found: error retention-days: 1 diff --git a/.github/workflows/prestissimo-worker-image-build.yml b/.github/workflows/prestissimo-worker-image-build.yml index eff622a372082..07e39c91e2e06 100644 --- a/.github/workflows/prestissimo-worker-image-build.yml +++ b/.github/workflows/prestissimo-worker-image-build.yml @@ -31,7 +31,7 @@ jobs: - name: "Download dependency image artifact" uses: "actions/download-artifact@v4" with: - name: "prestissimo-worker-dev-env-image-${{github.event.workflow_run.head_sha}}" + name: "prestissimo-worker-dev-env-image" path: ./_deps_image run-id: "${{github.event.workflow_run.id}}" github-token: "${{secrets.GITHUB_TOKEN}}" @@ -41,7 +41,7 @@ jobs: shell: bash run: | set -euo pipefail - TAR="./_deps_image/prestissimo-worker-dev-env-image-${{github.event.workflow_run.head_sha}}.tar" + TAR="./_deps_image/prestissimo-worker-dev-env-image.tar" OUT="$(docker load -i "$TAR")" echo "$OUT" IMAGE_REF="$(echo "$OUT" | awk -F': ' '/^Loaded image:/ {print $2}' | tail -n1)"