diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py index efe313b15..6e6afd421 100755 --- a/.buildkite/pipelines/build_linux.json.py +++ b/.buildkite/pipelines/build_linux.json.py @@ -36,7 +36,7 @@ "cpu": "6", "ephemeralStorage": "20G", "memory": "64G", - "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:33") + "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:34") }, "aarch64": { "provider": "aws", @@ -100,7 +100,7 @@ def main(args): "cpu": "6", "ephemeralStorage": "20G", "memory": "64G", - "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:16" + "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:17" }, "commands": [ ".buildkite/scripts/steps/build_and_test.sh" diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py index 681da7e13..4b72b6964 100755 --- a/.buildkite/pipelines/build_macos.json.py +++ b/.buildkite/pipelines/build_macos.json.py @@ -33,7 +33,7 @@ agents = { "aarch64": { "provider": "orka", - "image": "ml-macos-13-arm-004.orkasi" + "image": "ml-macos-13-arm-005.orkasi" } } envs = { diff --git a/.buildkite/pipelines/build_windows.json.py b/.buildkite/pipelines/build_windows.json.py index b357944f8..4872adcd2 100755 --- a/.buildkite/pipelines/build_windows.json.py +++ b/.buildkite/pipelines/build_windows.json.py @@ -45,7 +45,7 @@ def main(args): "provider": "gcp", "machineType": "c2-standard-16", "minCpuPlatform": "Intel Cascade Lake", - "image": "family/ml-cpp-4-windows-2022", + "image": "family/ml-cpp-5-windows-2022", }, "commands": [ f'if ( "{args.action}" -eq "debug" ) {{\$Env:ML_DEBUG="1"}}', diff --git a/.ci/orka/README.md b/.ci/orka/README.md index a85520183..6b9d39c36 100644 --- a/.ci/orka/README.md +++ b/.ci/orka/README.md @@ -9,7 +9,7 @@ ## Files - `install.sh` The script that does the software installs on the image -- `orka-macos-12-arm.pkr.hcl` The packer definition for a MacOS 12 ARM builder image +- `orka-macos-13-arm.pkr.hcl` The packer definition for a MacOS 13 ARM builder image ## Set Up Packer @@ -26,7 +26,7 @@ Make sure you are connected to the Orka VM. Packer requires access to secrets in vault, where VAULT_ADDR=https://vault-ci-prod.elastic.dev and VAULT_TOKEN must be set appropriately in the environment. -Run the following to create the image (MacOS 12 ARM in this example): +Run the following to create the image (MacOS 13 ARM in this example): ``` packer build orka-macos-13-arm.pkr.hcl @@ -63,7 +63,7 @@ The ML packer scripts do the following: * Install JDK `11.0.25` * Install CMake `3.30.5` * Install Boost `1.86.0` from source - * Install PyTorch `2.5.1` from source + * Install PyTorch `2.7.1` from source ## Caveats diff --git a/.ci/orka/orka-macos-13-arm.pkr.hcl b/.ci/orka/orka-macos-13-arm.pkr.hcl index bfe9dc96b..5ec0880f1 100644 --- a/.ci/orka/orka-macos-13-arm.pkr.hcl +++ b/.ci/orka/orka-macos-13-arm.pkr.hcl @@ -18,7 +18,7 @@ locals { source "macstadium-orka" "image" { source_image = "generic-13-ventura-arm-002.orkasi" - image_name = "ml-macos-13-arm-004.orkasi" + image_name = "ml-macos-13-arm-005.orkasi" orka_endpoint = local.orka_endpoint orka_user = local.orka_user orka_password = local.orka_password diff --git a/.ci/orka/third_party_deps.sh b/.ci/orka/third_party_deps.sh index 585edc511..b48bef31a 100644 --- a/.ci/orka/third_party_deps.sh +++ b/.ci/orka/third_party_deps.sh @@ -34,7 +34,7 @@ sudo rm -rf boost_1_86_0 sudo pip3 install numpy ninja pyyaml setuptools cffi typing_extensions future six requests dataclasses # Build and install PyTorch -git clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ +git clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -59,7 +59,7 @@ export DNNL_TARGET_ARCH=AARCH64 export USE_MKLDNN=ON export USE_QNNPACK=OFF export USE_PYTORCH_QNNPACK=OFF -export PYTORCH_BUILD_VERSION=2.5.1 +export PYTORCH_BUILD_VERSION=2.7.1 export PYTORCH_BUILD_NUMBER=1 python3 setup.py install diff --git a/3rd_party/licenses/pytorch-INFO.csv b/3rd_party/licenses/pytorch-INFO.csv index 6750bd122..04bd473a0 100644 --- a/3rd_party/licenses/pytorch-INFO.csv +++ b/3rd_party/licenses/pytorch-INFO.csv @@ -1,2 +1,2 @@ name,version,revision,url,license,copyright,sourceURL -PyTorch,2.5.1,a8d6afb511a69687bbb2b7e88a3cf67917e1697e,https://pytorch.org,BSD-3-Clause,, +PyTorch,2.7.1,e2d141dbde55c2a4370fac5165b0561b6af4798b,https://pytorch.org,BSD-3-Clause,, diff --git a/build-setup/linux.md b/build-setup/linux.md index 0621c2121..0fc99dffa 100644 --- a/build-setup/linux.md +++ b/build-setup/linux.md @@ -333,7 +333,7 @@ Then copy the shared libraries to the system directory: (cd /opt/intel/oneapi/mkl/2024.0 && tar cf - lib) | (cd /usr/local/gcc133 && sudo tar xvf -) ``` -### PyTorch 2.5.1 +### PyTorch 2.7.1 (This step requires a reasonable amount of memory. It failed on a machine with 8GB of RAM. It succeeded on a 16GB machine. You can specify the number of parallel jobs using environment variable MAX_JOBS. Lower number of jobs will reduce memory usage.) @@ -352,7 +352,7 @@ sudo /usr/local/gcc133/bin/python3.12 -m pip install numpy pyyaml setuptools cff Then obtain the PyTorch code: ``` -git clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git +git clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git cd pytorch git submodule sync git submodule update --init --recursive @@ -386,7 +386,7 @@ export USE_MKLDNN=ON export USE_QNNPACK=OFF export USE_PYTORCH_QNNPACK=OFF [ $(uname -m) = x86_64 ] && export USE_XNNPACK=OFF -export PYTORCH_BUILD_VERSION=2.5.1 +export PYTORCH_BUILD_VERSION=2.7.1 export PYTORCH_BUILD_NUMBER=1 /usr/local/gcc133/bin/python3.12 setup.py install ``` diff --git a/build-setup/macos.md b/build-setup/macos.md index 2ca1765ac..f34257ef6 100644 --- a/build-setup/macos.md +++ b/build-setup/macos.md @@ -133,18 +133,18 @@ Download the graphical installer for Python 3.10.10 from diff --git a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile index f485deca2..004892a09 100644 --- a/dev-tools/docker/linux_aarch64_cross_image/Dockerfile +++ b/dev-tools/docker/linux_aarch64_cross_image/Dockerfile @@ -27,7 +27,7 @@ RUN dnf -y update && \ RUN \ mkdir -p /usr/local/sysroot-aarch64-linux-gnu/usr && \ cd /usr/local/sysroot-aarch64-linux-gnu/usr && \ - wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-aarch64-linux-gnu-16.tar.bz2 | tar jxf - && \ + wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-aarch64-linux-gnu-17.tar.bz2 | tar jxf - && \ cd .. && \ ln -s usr/lib lib && \ ln -s usr/lib64 lib64 diff --git a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile index e4d03181d..1524b0045 100644 --- a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:16 +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:17 MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_aarch64_native_image/Dockerfile b/dev-tools/docker/linux_aarch64_native_image/Dockerfile index f3cb5d19c..7e8bd99c0 100644 --- a/dev-tools/docker/linux_aarch64_native_image/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_image/Dockerfile @@ -169,7 +169,7 @@ RUN \ # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -183,7 +183,7 @@ RUN \ export USE_MKLDNN=ON && \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ export MAX_JOBS=10 && \ /usr/local/bin/python3.12 setup.py install && \ diff --git a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile index aa3c6c33c..008376518 100644 --- a/dev-tools/docker/linux_aarch64_native_tester/Dockerfile +++ b/dev-tools/docker/linux_aarch64_native_tester/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:16 +FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:17 MAINTAINER David Roberts diff --git a/dev-tools/docker/linux_builder/Dockerfile b/dev-tools/docker/linux_builder/Dockerfile index 587229e07..ad5ab9483 100644 --- a/dev-tools/docker/linux_builder/Dockerfile +++ b/dev-tools/docker/linux_builder/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:33 +FROM docker.elastic.co/ml-dev/ml-linux-build:34 LABEL maintainer="Valeriy Khakhutskyy " diff --git a/dev-tools/docker/linux_dependency_builder_image/Dockerfile b/dev-tools/docker/linux_dependency_builder_image/Dockerfile index bab0880c0..1f172c33b 100644 --- a/dev-tools/docker/linux_dependency_builder_image/Dockerfile +++ b/dev-tools/docker/linux_dependency_builder_image/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:33 AS builder +FROM docker.elastic.co/ml-dev/ml-linux-build:34 AS builder # This is basically automating the setup instructions in build-setup/linux.md diff --git a/dev-tools/docker/linux_image/Dockerfile b/dev-tools/docker/linux_image/Dockerfile index 34426f99e..7099e1a6d 100644 --- a/dev-tools/docker/linux_image/Dockerfile +++ b/dev-tools/docker/linux_image/Dockerfile @@ -183,7 +183,7 @@ gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS. # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -198,7 +198,7 @@ RUN \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ export USE_XNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ export MAX_JOBS=10 && \ /usr/local/bin/python3.12 setup.py install && \ diff --git a/dev-tools/docker/linux_tester/Dockerfile b/dev-tools/docker/linux_tester/Dockerfile index 058709cae..7e3ef0ba5 100644 --- a/dev-tools/docker/linux_tester/Dockerfile +++ b/dev-tools/docker/linux_tester/Dockerfile @@ -10,7 +10,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-linux-build:33 +FROM docker.elastic.co/ml-dev/ml-linux-build:34 LABEL maintainer="Valeriy Khakhutskyy " diff --git a/dev-tools/docker/ubuntu_devbox/Dockerfile b/dev-tools/docker/ubuntu_devbox/Dockerfile index 4da7076a9..0d5a7becb 100644 --- a/dev-tools/docker/ubuntu_devbox/Dockerfile +++ b/dev-tools/docker/ubuntu_devbox/Dockerfile @@ -165,7 +165,7 @@ ENV PATH=/usr/local/gcc133/bin:/usr/bin:/bin:/usr/sbin:/sbin # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION RUN \ cd ${build_dir} && \ - git -c advice.detachedHead=false clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \ + git -c advice.detachedHead=false clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \ cd pytorch && \ git submodule sync && \ git submodule update --init --recursive && \ @@ -179,7 +179,7 @@ RUN \ export USE_QNNPACK=OFF && \ export USE_PYTORCH_QNNPACK=OFF && \ export USE_XNNPACK=OFF && \ - export PYTORCH_BUILD_VERSION=2.5.1 && \ + export PYTORCH_BUILD_VERSION=2.7.1 && \ export PYTORCH_BUILD_NUMBER=1 && \ export MAX_JOBS=10 && \ /usr/local/bin/python3.12 setup.py install diff --git a/dev-tools/docker_build.sh b/dev-tools/docker_build.sh index fc1eac34a..5edc131f6 100755 --- a/dev-tools/docker_build.sh +++ b/dev-tools/docker_build.sh @@ -80,7 +80,7 @@ do TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$ prefetch_docker_base_image "$DOCKERFILE" - docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT --build-arg ML_DEBUG=$ML_DEBUG -f "$DOCKERFILE" . + docker build --no-cache --force-rm -t $TEMP_TAG --progress=plain --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT --build-arg ML_DEBUG=$ML_DEBUG -f "$DOCKERFILE" . # Using tar to copy the build artifacts out of the container seems more reliable # than docker cp, and also means the files end up with the correct uid/gid docker run --rm --workdir=/ml-cpp $TEMP_TAG bash -c "tar cf - build/distributions && sleep 30" | tar xvf - diff --git a/dev-tools/docker_test.sh b/dev-tools/docker_test.sh index 22a46f4c9..d777e7203 100755 --- a/dev-tools/docker_test.sh +++ b/dev-tools/docker_test.sh @@ -88,7 +88,7 @@ do TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$ prefetch_docker_base_image "$DOCKERFILE" - docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT --build-arg BOOST_TEST_OUTPUT_FORMAT_FLAGS=$BOOST_TEST_OUTPUT_FORMAT_FLAGS --build-arg ML_DEBUG=$ML_DEBUG -f "$DOCKERFILE" . + docker build --no-cache --force-rm -t $TEMP_TAG --progress=plain --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT --build-arg BOOST_TEST_OUTPUT_FORMAT_FLAGS=$BOOST_TEST_OUTPUT_FORMAT_FLAGS --build-arg ML_DEBUG=$ML_DEBUG -f "$DOCKERFILE" . # Using tar to copy the build and test artifacts out of the container seems # more reliable than docker cp, and also means the files end up with the # correct uid/gid diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 3b119797c..6340e2e5d 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -27,6 +27,11 @@ //=== Bug Fixes //=== Regressions +== {es} version 9.2.0 + +=== Enhancements + +* Update the PyTorch library to version 2.7.1. (See {ml-pull}2863[#2863].) == {es} version 9.1.0