Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions .github/workflows/prestissimo-worker-dep-image-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
name: "prestissimo-worker-dep-image-build"

on:
pull_request:
push:

jobs:
filter-relevant-changes:
name: "filter-relevant-changes"
runs-on: "ubuntu-24.04"
outputs:
prestissimo_dependency_changed: "${{steps.filter.outputs.prestissimo_dependency}}"
steps:
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
with:
submodules: "recursive"

- name: "Work around actions/runner-images/issues/6775"
run: "chown $(id -u):$(id -g) -R ."
shell: "bash"

- name: "Filter relevant changes"
uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36"
id: "filter"
with:
base: "release-0.293-clp-connector"
filters: |
prestissimo_dependency:
- "presto-native-execution/scripts/**/*"
- "presto-native-execution/velox/scripts/**/*"

prestissimo-worker-dep-image-build:
name: "prestissimo-worker-dep-image-build"
runs-on: "ubuntu-22.04"
env:
SHOULD_PUSH: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector' }}
steps:
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
with:
submodules: "recursive"

- name: "Login to image registry"
uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772"
with:
registry: "ghcr.io"
username: "${{github.actor}}"
password: "${{secrets.GITHUB_TOKEN}}"

- name: "Set up Docker Buildx"
uses: "docker/setup-buildx-action@v3"

- name: "Set up metadata for dependency image"
id: "metadata-deps-image"
uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"
with:
images: "ghcr.io/${{github.repository}}/prestissimo-worker-dev-env"
tags: "type=raw,value=dev"

- name: "Build and push dependency image"
if: "'true' == needs.filter-relevant-changes.outputs.prestissimo_dependency_changed"
uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"
with:
context: "./presto-native-execution"
file: "./presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile"
push: ${{ 'true' == env.SHOULD_PUSH }}
tags: "${{steps.metadata-deps-image.outputs.tags}}"
labels: "${{steps.metadata-deps-image.outputs.labels}}"

- name: "Pack dependency image as tar"
run: |
if [[ 'true' == "${{ needs.filter-relevant-changes.outputs.prestissimo_dependency_changed }}" ]]; then
SRC_IMAGE="${{ steps.metadata-deps-image.outputs.tags }}"
else
SRC_IMAGE="ghcr.io/y-scope/presto/prestissimo-worker-dev-env:dev"
docker pull "$SRC_IMAGE"
fi

TAR="prestissimo-worker-dev-env-image.tar"
docker save "$SRC_IMAGE" -o "$TAR"
echo "Saved $SRC_IMAGE to $TAR"

- name: "Upload dependency image artifact"
uses: "actions/upload-artifact@v4"
with:
name: prestissimo-worker-dev-env-image
path: prestissimo-worker-dev-env-image.tar
if-no-files-found: error
retention-days: 1
92 changes: 92 additions & 0 deletions .github/workflows/prestissimo-worker-image-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
name: "prestissimo-worker-image-build"

on:
workflow_run:
workflows: [ "prestissimo-worker-dep-image-build" ] # name of the producer workflow
types: [ completed ]
pull_request:
push:

jobs:
prestissimo-worker-image-build:
name: "prestissimo-worker-image-build"
runs-on: "ubuntu-22.04"
env:
SHOULD_PUSH: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector' }}
steps:
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
with:
submodules: "recursive"

- name: "Login to image registry"
uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772"
with:
registry: "ghcr.io"
username: "${{github.actor}}"
password: "${{secrets.GITHUB_TOKEN}}"

- name: "Set up Docker Buildx"
uses: "docker/setup-buildx-action@v3"

- name: "Download dependency image artifact"
uses: "actions/download-artifact@v4"
with:
name: "prestissimo-worker-dev-env-image"
path: ./_deps_image
run-id: "${{github.event.workflow_run.id}}"
github-token: "${{secrets.GITHUB_TOKEN}}"

- name: "Load dependency image into Docker"
id: "load-deps"
shell: bash
run: |
set -euo pipefail
TAR="./_deps_image/prestissimo-worker-dev-env-image.tar"
OUT="$(docker load -i "$TAR")"
echo "$OUT"
IMAGE_REF="$(echo "$OUT" | awk -F': ' '/^Loaded image:/ {print $2}' | tail -n1)"

# Fallback if the tar had no tag and you only get an image ID:
if [[ -z "${IMAGE_REF:-}" ]]; then
ID="$(echo "$OUT" | sed -n 's/^Loaded image ID: //p' | tail -n1)"
IMAGE_REF="$(docker image inspect --format='{{index .RepoTags 0}}' "$ID" 2>/dev/null || true)"
fi

if [[ -z "${IMAGE_REF:-}" ]]; then
echo "Could not determine image tag from docker load output" >&2
exit 1
fi

echo "image_ref=$IMAGE_REF" >> "$GITHUB_OUTPUT"
echo "Loaded ${IMAGE_REF} image."

- name: "Set up metadata for runtime image"
id: "metadata-runtime-image"
uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"
with:
images: "ghcr.io/${{github.repository}}/prestissimo-worker"
tags: "type=raw,value=dev"

- name: "Get number of cores"
id: "get-cores"
run: |-
echo "num_cores=$(nproc)" >> $GITHUB_OUTPUT

- name: "Build and push runtime image"
uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"
with:
build-args: |-
BASE_IMAGE=ubuntu:22.04
DEPENDENCY_IMAGE=${{steps.load-deps.outputs.image_ref}}
EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF \
-DPRESTO_ENABLE_PARQUET=ON \
-DPRESTO_ENABLE_S3=ON
NUM_THREADS=${{steps.get-cores.outputs.num_cores}}
OSNAME=ubuntu
context: "./presto-native-execution"
file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile"
push: >-
${{github.event_name != 'pull_request'
&& github.ref == 'refs/heads/release-0.293-clp-connector'}}
tags: "${{steps.metadata-runtime-image.outputs.tags}}"
labels: "${{steps.metadata-runtime-image.outputs.labels}}"
70 changes: 0 additions & 70 deletions .github/workflows/prestissimo-worker-images-build.yml

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
using namespace facebook::presto;
using namespace facebook::velox;

class PrestoToVeloxConnectorTest : public ::testing::Test {};
class PrestoToVeloxConnectorTest : public ::testing::Test {};

TEST_F(PrestoToVeloxConnectorTest, registerVariousConnectors) {
std::vector<std::pair<std::string, std::unique_ptr<PrestoToVeloxConnector>>>
Expand Down
Loading