Merge pull request #1323 from tensorflow/test_692953635

Add docker build scripts and enable aarch64 pip wheels.

Author: cantonios

WORKSPACE

@@ -110,6 +110,7 @@ python_init_repositories(
         "3.9": "//oss_scripts/pip_package:requirements_lock_3_9.txt",
         "3.10": "//oss_scripts/pip_package:requirements_lock_3_10.txt",
         "3.11": "//oss_scripts/pip_package:requirements_lock_3_11.txt",
+        "3.12": "//oss_scripts/pip_package:requirements_lock_3_12.txt",
     },
     default_python_version = "system",
 )

oss_scripts/build.Dockerfile.aarch64

@@ -0,0 +1,37 @@
+# Constructs the environment within which we will build the tensorflow-text
+# pip wheels.
+
+FROM linaro/tensorflow-arm64-build:2.16-multipython
+LABEL maintainer="TensorFlow-Text team <tf-text-team@google.com>"
+
+ARG PYTHON_VERSION
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHON_BIN_PATH=/usr/bin/python${PYTHON_VERSION}
+
+# Install supplementary Python interpreters
+RUN ln -s ${PYTHON_BIN_PATH} /usr/local/bin/python && \
+    ln -s ${PYTHON_BIN_PATH} /usr/local/bin/python3 && \
+    ln -s ${PYTHON_BIN_PATH} /usr/bin/python
+
+RUN --mount=type=cache,target=/var/cache/apt \
+  apt update && \
+  apt install -yqq \
+    apt-utils \
+    build-essential \
+    checkinstall \
+    libffi-dev
+
+# Install pip dependencies needed for tensorflow-text
+RUN --mount=type=cache,target=/root/.cache \
+  ${PYTHON_BIN_PATH} -m pip install -U pip && \
+  ${PYTHON_BIN_PATH} -m pip install -U \
+    absl-py \
+    auditwheel \
+    etils[epath] \
+    patchelf \
+    setuptools \
+    twine \
+    wheel;
+
+WORKDIR "/tmp/tensorflow_text"

oss_scripts/build.Dockerfile.x86_64

@@ -0,0 +1,32 @@
+# Constructs the environment within which we will build the tensorflow-text
+# pip wheels.
+
+ARG PYTHON_VERSION
+FROM tensorflow/build:2.18-python${PYTHON_VERSION}
+LABEL maintainer="TensorFlow-Text team <tf-text-team@google.com>"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install supplementary Python interpreters
+RUN mkdir /tmp/python
+RUN --mount=type=cache,target=/var/cache/apt \
+  apt update && \
+  apt install -yqq \
+    apt-utils \
+    build-essential \
+    checkinstall \
+    libffi-dev
+
+# Install pip dependencies needed for tensorflow text.
+RUN --mount=type=cache,target=/root/.cache \
+  python${PYTHON_VERSION} -m pip install -U pip && \
+  python${PYTHON_VERSION} -m pip install -U \
+    absl-py \
+    auditwheel \
+    etils[epath] \
+    patchelf \
+    setuptools \
+    twine \
+    wheel;
+
+WORKDIR "/tmp/tensorflow_text"

oss_scripts/build_docs.py

@@ -13,6 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Copyright 2024 TF.Text Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 r"""Tool to generate external api_docs.
 
 python build_docs.py --output_dir=/tmp/text_api

oss_scripts/configure.sh

@@ -38,15 +38,10 @@ if [[ $(pip show tensorflow) == *tensorflow* ]] ||
   echo 'Using installed tensorflow.'
 else
   echo 'Installing tensorflow.'
-  if is_macos; then
-    #  Only Apple Silicon will be installed with tensorflow-macos.
-    if [[ x"$(arch)" == x"arm64" ]]; then
-      pip install tensorflow-macos==2.16.1
-    else
-      pip install tensorflow==2.16.1
-    fi
+  if [[ "$IS_NIGHTLY" == "nightly" ]]; then
+    pip install tf-nightly
   else
-    pip install tensorflow==2.16.1
+    pip install tensorflow==2.18.0
   fi
 fi
 
@@ -58,8 +53,6 @@ curl https://raw.githubusercontent.com/tensorflow/tensorflow/master/.bazelrc -o
 # This line breaks Windows builds, so we remove it.
 sed -i -e 's/build --noincompatible_remove_legacy_whole_archive//' .bazelrc
 
-# the next line is temporary to aid in transition
-write_to_bazelrc "build:manylinux2010 --config=release_cpu_linux"
 write_to_bazelrc "build:manylinux2014 --config=release_cpu_linux"
 
 if (which python3) | grep -q "python3"; then

oss_scripts/docker_builds.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# This script builds a docker container, and pip wheels for all supported
+# Python versions.  Run from the root tensorflow-text directory:
+#
+# ./oss_scripts/docker_builds.sh
+
+set -e -x
+
+# If a specific PYTHON_VERSION is specified, only build that one.
+# Otherwisebuild all supported versions.
+python_versions=("3.9" "3.10" "3.11" "3.12")
+if [[ ! -z ${PYTHON_VERSION+x} ]]; then
+  python_versions=("$PYTHON_VERSION")
+fi
+
+# Clean previous images.
+for python_version in ${python_versions[@]}
+do
+  docker rmi -f tensorflow_text:${python_version} || true
+done
+
+arch=$(uname -m)
+build_args=()
+if [ "$arch" == "x86_64" ]; then
+  build_args+=("--config=release_cpu_linux")
+  build_args+=("--platforms=@sigbuild-r2.17-clang_config_platform//:platform")
+  auditwheel_platform="manylinux2014_x86_64"
+elif [ "$arch" == "aarch64" ]; then
+  build_args+=("--crosstool_top=@ml2014_aarch64_config_aarch64//crosstool:toolchain")
+  auditwheel_platform="manylinux2014_aarch64"
+fi
+
+# Build wheel for each Python version.
+for python_version in ${python_versions[@]}
+do
+  DOCKER_BUILDKIT=1 docker build --progress=plain --no-cache \
+    --build-arg HERMETIC_PYTHON_VERSION=${python_version} --build-arg PYTHON_VERSION=${python_version} \
+    -t tensorflow_text:${python_version} - < "oss_scripts/build.Dockerfile.${arch}"
+
+  docker run --rm -a stdin -a stdout -a stderr \
+    --env PYTHON_VERSION=${python_version} \
+    --env HERMETIC_PYTHON_VERSION=${python_version} \
+    --env BUILD_ARGS=${build_args} \
+    --env AUDITWHEEL_PLATFORM=${auditwheel_platform} \
+    --env IS_NIGHTLY=${IS_NIGHTLY} \
+    -v $PWD:/tmp/tensorflow_text \
+    --name tensorflow_text tensorflow_text:${python_version} \
+    bash oss_scripts/run_build.sh
+done

oss_scripts/pip_package/requirements.in

@@ -1,3 +1,5 @@
+# ai-edge-litert
+setuptools==70.0.0
 tensorflow
 tf-keras
 tensorflow-datasets