From 3f6da5228b67d83d6cefe29c6927a6cca3c2f285 Mon Sep 17 00:00:00 2001 From: Christian Zentgraf Date: Wed, 22 Oct 2025 11:32:22 -0700 Subject: [PATCH] build(native): Refactor setup scripts A recent change in Velox changed what the current directory is after github_checkout. This breaks building the PrestoC++ dependencies. The refactor removes github cloning and instead pulls released tar.gz images of the source code. This also increases the download speed. --- .../scripts/setup-adapters.sh | 13 ++++---- .../scripts/setup-centos.sh | 32 +++++++++---------- .../scripts/setup-macos.sh | 26 +++++---------- .../scripts/setup-ubuntu.sh | 4 +-- 4 files changed, 32 insertions(+), 43 deletions(-) diff --git a/presto-native-execution/scripts/setup-adapters.sh b/presto-native-execution/scripts/setup-adapters.sh index 42596d1c059b1..c5a1721e87815 100755 --- a/presto-native-execution/scripts/setup-adapters.sh +++ b/presto-native-execution/scripts/setup-adapters.sh @@ -14,6 +14,9 @@ # Propagate errors and improve debugging. set -eufx -o pipefail +JWT_VERSION="v0.6.0" +PROMETHEUS_VERSION="v1.2.4" + SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") if [ -f "${SCRIPT_DIR}/setup-common.sh" ] then @@ -31,15 +34,13 @@ else fi function install_jwt_cpp { - github_checkout Thalhammer/jwt-cpp v0.6.0 --depth 1 - cmake_install -DBUILD_TESTS=OFF -DJWT_BUILD_EXAMPLES=OFF -DJWT_DISABLE_PICOJSON=ON -DJWT_CMAKE_FILES_INSTALL_DIR="${DEPENDENCY_DIR}/jwt-cpp" + wget_and_untar https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/${JWT_VERSION}.tar.gz jwt-cpp + cmake_install_dir jwt-cpp -DBUILD_TESTS=OFF -DJWT_BUILD_EXAMPLES=OFF -DJWT_DISABLE_PICOJSON=ON -DJWT_CMAKE_FILES_INSTALL_DIR="${INSTALL_PREFIX}/jwt-cpp" } function install_prometheus_cpp { - github_checkout jupp0r/prometheus-cpp v1.2.4 --depth 1 - git submodule init - git submodule update - cmake_install -DBUILD_SHARED_LIBS=ON -DENABLE_PUSH=OFF -DENABLE_COMPRESSION=OFF + wget_and_untar https://github.com/jupp0r/prometheus-cpp/releases/download/${PROMETHEUS_VERSION}/prometheus-cpp-with-submodules.tar.gz prometheus-cpp + cmake_install_dir prometheus-cpp -DBUILD_SHARED_LIBS=ON -DENABLE_PUSH=OFF -DENABLE_COMPRESSION=OFF } function install_arrow_flight { diff --git a/presto-native-execution/scripts/setup-centos.sh b/presto-native-execution/scripts/setup-centos.sh index 20728ffd5851a..523a35f4f2bfe 100755 --- a/presto-native-execution/scripts/setup-centos.sh +++ b/presto-native-execution/scripts/setup-centos.sh @@ -17,7 +17,7 @@ set -x export CC=/opt/rh/gcc-toolset-12/root/bin/gcc export CXX=/opt/rh/gcc-toolset-12/root/bin/g++ -WGET_OPTIONS=${WGET_OPTIONS:-""} +GPERF_VERSION="3.1" CPU_TARGET="${CPU_TARGET:-avx}" SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") @@ -38,28 +38,26 @@ function install_presto_deps_from_package_managers { } function install_gperf { - wget ${WGET_OPTIONS} https://ftp.gnu.org/pub/gnu/gperf/gperf-3.1.tar.gz || - wget ${WGET_OPTIONS} https://mirrors.ocf.berkeley.edu/gnu/gperf/gperf-3.1.tar.gz && - tar -xzf gperf-3.1.tar.gz && - cd gperf-3.1 && - ./configure --prefix=/usr/local/gperf/3_1 && - make "-j${NPROC}" && - make install - if [ -f /usr/local/bin/gperf ]; then - echo "Did not create '/usr/local/bin/gperf' symlink as file already exists." - else - ln -s /usr/local/gperf/3_1/bin/gperf /usr/local/bin/ - fi + wget_and_untar https://mirrors.ocf.berkeley.edu/gnu/gperf/gperf-${GPERF_VERSION}.tar.gz gperf + ( + cd ${DEPENDENCY_DIR}/gperf || exit && + ./configure --prefix=/usr/local/gperf/3_1 && + make "-j${NPROC}" && + make install + if [ -f /usr/local/bin/gperf ]; then + echo "Did not create '/usr/local/bin/gperf' symlink as file already exists." + else + ln -s /usr/local/gperf/3_1/bin/gperf /usr/local/bin/ + fi + ) } function install_proxygen { - git clone https://github.com/facebook/proxygen - cd proxygen - git checkout $FB_OS_VERSION + wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen # Folly Portability.h being used to decide whether or not support coroutines # causes issues (build, lin) if the selection is not consistent across users of folly. EXTRA_PKG_CXXFLAGS=" -DFOLLY_CFG_NO_COROUTINES" - cmake_install -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON + cmake_install_dir proxygen -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON } function install_presto_deps { diff --git a/presto-native-execution/scripts/setup-macos.sh b/presto-native-execution/scripts/setup-macos.sh index b05135c57e557..4f031e6b125e9 100755 --- a/presto-native-execution/scripts/setup-macos.sh +++ b/presto-native-execution/scripts/setup-macos.sh @@ -26,30 +26,20 @@ source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-macos.sh" GPERF_VERSION="3.1" function install_proxygen { - github_checkout facebook/proxygen "${FB_OS_VERSION}" --depth 1 + wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen # Folly Portability.h being used to decide whether or not support coroutines # causes issues (build, lin) if the selection is not consistent across users of folly. EXTRA_PKG_CXXFLAGS=" -DFOLLY_CFG_NO_COROUTINES" - cmake_install -DBUILD_TESTS=OFF + cmake_install_dir proxygen -DBUILD_TESTS=OFF } function install_gperf { - mkdir -p "${DEPENDENCY_DIR}" - cd "${DEPENDENCY_DIR}" - if [ -d gperf ]; then - if prompt "gperf already exists. Delete?"; then - ${SUDO} rm -rf gperf - else - return - fi - fi - (curl ${CURL_OPTIONS:+${CURL_OPTIONS}} -L https://ftp.gnu.org/pub/gnu/gperf/gperf-${GPERF_VERSION}.tar.gz -o gperf-${GPERF_VERSION}.tar.gz || - curl ${CURL_OPTIONS:+${CURL_OPTIONS}} -L https://mirrors.ocf.berkeley.edu/gnu/gperf/gperf-${GPERF_VERSION}.tar.gz -o gperf-${GPERF_VERSION}.tar.gz) && - mkdir -p gperf && - tar -xz --strip-components=1 --no-same-owner -f gperf-${GPERF_VERSION}.tar.gz -C gperf && - cd gperf && - ./configure --prefix=${INSTALL_PREFIX} && - make install + wget_and_untar https://mirrors.ocf.berkeley.edu/gnu/gperf/gperf-${GPERF_VERSION}.tar.gz gperf + ( + cd ${DEPENDENCY_DIR}/gperf || exit && + ./configure --prefix=${INSTALL_PREFIX} && + make install + ) } function install_presto_deps { diff --git a/presto-native-execution/scripts/setup-ubuntu.sh b/presto-native-execution/scripts/setup-ubuntu.sh index f36661e1b9016..947f11ded94e7 100755 --- a/presto-native-execution/scripts/setup-ubuntu.sh +++ b/presto-native-execution/scripts/setup-ubuntu.sh @@ -24,11 +24,11 @@ function install_proxygen { # proxygen requires python and gperf ${SUDO} apt update ${SUDO} apt install -y gperf python3 - github_checkout facebook/proxygen "${FB_OS_VERSION}" --depth 1 + wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen # Folly Portability.h being used to decide whether or not support coroutines # causes issues (build, lin) if the selection is not consistent across users of folly. EXTRA_PKG_CXXFLAGS=" -DFOLLY_CFG_NO_COROUTINES" - cmake_install -DBUILD_TESTS=OFF + cmake_install_dir proxygen -DBUILD_TESTS=OFF } function install_presto_deps {