diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index a184df2ec42..fafb45431f4 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -269,6 +269,11 @@ jobs: brew install ccache ccache -M 2G # See .github/workflows/readme.md for ccache strategy. + - name: Install Open3D-ML Python dependencies + working-directory: ${{ env.OPEN3D_ML_ROOT }} + run: | + python -m pip install -r requirements.txt + - name: Config and build wheel run: | PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 043cb0c7b97..9871c13eed3 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -27,7 +27,7 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - pip install -U clang-format~=10.0.0 yapf==0.30.0 nbformat + pip install -U -r python/requirements_style.txt - name: Run style check run: | python util/check_style.py diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 04a408802d0..8fab5818504 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -21,8 +21,6 @@ concurrency: env: PIP_VER: "23.2.1" - WHEEL_VER: "0.38.4" - STOOLS_VER: "67.3.2" JEDI_VER: "0.17.2" # https://github.com/ipython/ipython/issues/12740 IDNA_VER: "2.8" # https://github.com/psf/requests/issues/5710 CUDA_VERSION: "12.1.0" @@ -50,6 +48,7 @@ jobs: - BUILD_CUDA_MODULE: ON env: BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} + WITH_STUBGEN: ${{ ( matrix.BUILD_SHARED_LIBS == 'ON' || matrix.BUILD_CUDA_MODULE == 'ON' ) && 'OFF' || 'ON' }} steps: - name: Disk space used @@ -123,6 +122,7 @@ jobs: -DBUILD_WEBRTC=${{ env.BUILD_WEBRTC }} ` -DBUILD_UNIT_TESTS=ON ` -DBUILD_CUDA_MODULE=${{ matrix.BUILD_CUDA_MODULE }} ` + -DWITH_STUBGEN=${{ env.WITH_STUBGEN }} ` ${{ env.SRC_DIR }} - name: Build @@ -213,15 +213,23 @@ jobs: cmake --build . --config ${{ matrix.CONFIG }} .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test Remove-Item "C:\Program Files\Open3D" -Recurse + - name: Install Open3D python build requirements + working-directory: ${{ env.SOURCE_DIR }} + run: | + $ErrorActionPreference = 'Stop' + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -r python/requirements_build.txt + python -m pip install -U jedi==${{ env.JEDI_VER }} idna==${{ env.IDNA_VER }} + - name: Install Open3D python requirements for stubgen + working-directory: ${{ env.SOURCE_DIR }} + if: ${{ env.WITH_STUBGEN == 'ON' }} + run: | + $ErrorActionPreference = 'Stop' + python -m pip install -U -r python/requirements.txt - name: Install Python package working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} ` - jedi==${{ env.JEDI_VER }} ` - idna==${{ env.IDNA_VER }} cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package - name: Import python package # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python. @@ -272,7 +280,9 @@ jobs: working-directory: ${{ env.SRC_DIR }} run: | $ErrorActionPreference = 'Stop' + python -m pip install -U pip==${{ env.PIP_VER }} python -m pip install -r python/requirements.txt + python -m pip install -r python/requirements_build.txt python -m pip install -r python/requirements_jupyter_build.txt - name: Config @@ -300,9 +310,6 @@ jobs: working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} cmake --build . --parallel ${{ env.NPROC }} --config Release --target pip-package $PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name echo "PIP_PKG_NAME=$PIP_PKG_NAME" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append @@ -366,9 +373,8 @@ jobs: python -m venv open3d_test_venv open3d_test_venv\Scripts\Activate.ps1 - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -r python/requirements_build.txt python -m pip install -U -r python/requirements_test.txt $py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") if (Test-Path -Path "pip_package") { diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1f4f43b17..8c71e011c0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ - Fix render to depth image on Apple Retina displays (PR #7001) - Fix infinite loop in segment_plane if num_points < ransac_n (PR #7032) - Add select_by_index method to Feature class (PR #7039) +- Include typing stubs in python package (PR #6917) ## 0.13 diff --git a/CMakeLists.txt b/CMakeLists.txt index e1bd4706288..2433ecee515 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,8 @@ option(BUILD_EXAMPLES "Build Open3D examples programs" ON option(BUILD_UNIT_TESTS "Build Open3D unit tests" OFF) option(BUILD_BENCHMARKS "Build the micro benchmarks" OFF) option(BUILD_PYTHON_MODULE "Build the python module" ON ) +option(WITH_STUBGEN "Use pybind11-stubgen to generate stubs" ON ) +option(IGNORE_STUBGEN_ERRORS "Ignore all errors during stub generation" ON ) option(BUILD_CUDA_MODULE "Build the CUDA module" OFF) option(BUILD_WITH_CUDA_STATIC "Build with static CUDA libraries" ON ) option(BUILD_COMMON_CUDA_ARCHS "Build for common CUDA GPUs (for release)" OFF) diff --git a/cmake/Open3DPrintConfigurationSummary.cmake b/cmake/Open3DPrintConfigurationSummary.cmake index 3e3d42da11e..bd4cfc700bc 100644 --- a/cmake/Open3DPrintConfigurationSummary.cmake +++ b/cmake/Open3DPrintConfigurationSummary.cmake @@ -37,6 +37,10 @@ function(open3d_print_configuration_summary) open3d_aligned_print("Build Unit Tests" "${BUILD_UNIT_TESTS}") open3d_aligned_print("Build Examples" "${BUILD_EXAMPLES}") open3d_aligned_print("Build Python Module" "${BUILD_PYTHON_MODULE}") + open3d_aligned_print("Generate Typing Stubs" "${WITH_STUBGEN}") + if(WITH_STUBGEN) + open3d_aligned_print("Ignore All Stubgen Errors" "${IGNORE_STUBGEN_ERRORS}") + endif() open3d_aligned_print("Build Jupyter Extension" "${BUILD_JUPYTER_EXTENSION}") open3d_aligned_print("Build TensorFlow Ops" "${BUILD_TENSORFLOW_OPS}") open3d_aligned_print("Build PyTorch Ops" "${BUILD_PYTORCH_OPS}") diff --git a/cpp/pybind/CMakeLists.txt b/cpp/pybind/CMakeLists.txt index 6efae9a17fd..b569119835a 100644 --- a/cpp/pybind/CMakeLists.txt +++ b/cpp/pybind/CMakeLists.txt @@ -226,6 +226,8 @@ add_custom_target(python-package -DPYTHON_VERSION=${PYTHON_VERSION} "-DCOMPILED_MODULE_PATH_LIST=${COMPILED_MODULE_PATH_LIST}" "-DPYTHON_EXTRA_LIBRARIES=${PYTHON_EXTRA_LIBRARIES}" + -DWITH_STUBGEN=${WITH_STUBGEN} + -DIGNORE_STUBGEN_ERRORS=${IGNORE_STUBGEN_ERRORS} -DBUILD_JUPYTER_EXTENSION=${BUILD_JUPYTER_EXTENSION} -DBUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS} -DBUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS} diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index 01e0d5663f5..d1b38659770 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -136,3 +136,17 @@ file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples") file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples") + +# Generate typing stub files (.pyi) and py.typed marker file. +if(WITH_STUBGEN) + if(NOT IGNORE_STUBGEN_ERRORS) + list(APPEND PYBIND11_STUBGEN_FLAGS "--exit-code") + endif() + message(STATUS "Generating typing stubs...") + execute_process( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHON_PACKAGE_DST_DIR} pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR} ${PYBIND11_STUBGEN_FLAGS} + COMMAND_ECHO STDOUT + COMMAND_ERROR_IS_FATAL ANY + ) + file(WRITE "${PYTHON_PACKAGE_DST_DIR}/open3d/py.typed") +endif() diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index eb60286b620..edf30a1015c 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -109,11 +109,15 @@ RUN /root/Open3D/util/install_deps_ubuntu.sh assume-yes \ # Open3D Python dependencies COPY ./util/ci_utils.sh /root/Open3D/util/ COPY ./python/requirements.txt /root/Open3D/python/ +COPY ./python/requirements_build.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/ RUN source /root/Open3D/util/ci_utils.sh \ && install_python_dependencies with-jupyter +# Open3D-ML Python dependencies +RUN python -m pip install -r "${OPEN3D_ML_ROOT}/requirements.txt" + # Open3D Jupyter dependencies RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ && apt-get install -y nodejs \ diff --git a/python/open3d/__init__.py b/python/open3d/__init__.py index d4d5c613b0e..5a8c11556fb 100644 --- a/python/open3d/__init__.py +++ b/python/open3d/__init__.py @@ -212,4 +212,4 @@ def _jupyter_nbextension_paths(): if sys.platform == "win32": _win32_dll_dir.close() -del os, sys, CDLL, load_cdll, find_library, Path, warnings, _insert_pybind_names +del os, sys, CDLL, load_cdll, find_library, Path, warnings, _insert_pybind_names, open3d diff --git a/python/pyproject.toml b/python/pyproject.toml index 3d229011876..f65c9888bcb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,3 +1,12 @@ [build-system] -requires = ["ipywidgets>=8.0.3", "pygments>=2.7.4", "jupyter_packaging~=0.12", "jupyterlab>=3.0.0,==3.*", "setuptools>=50.3.2", "wheel==0.38.4"] +requires = [ + "ipywidgets>=8.0.4", + "pygments>=2.7.4", + "jupyter_packaging~=0.12", + "jupyterlab>=3.0.0,==3.*", + "setuptools>=67.3.2", + "wheel==0.38.4", + "yapf==0.30.0", + "pybind11-stubgen==2.5.1", +] build-backend = "setuptools.build_meta" diff --git a/python/requirements_build.txt b/python/requirements_build.txt index b9bb3b1f28a..9d8dc63cb4b 100644 --- a/python/requirements_build.txt +++ b/python/requirements_build.txt @@ -1,3 +1,4 @@ setuptools>=67.3.2 wheel==0.38.4 yapf==0.30.0 +pybind11-stubgen==2.5.1 \ No newline at end of file diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 1adb4b7d8c5..8c8f4217361 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -30,9 +30,6 @@ TORCH_VER="2.2.2" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python PIP_VER="23.2.1" -WHEEL_VER="0.38.4" -STOOLS_VER="67.3.2" -YAPF_VER="0.30.0" PROTOBUF_VER="4.24.0" OPEN3D_INSTALL_DIR=~/open3d_install @@ -42,10 +39,10 @@ install_python_dependencies() { echo "Installing Python dependencies" options="$(echo "$@" | tr ' ' '|')" - python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ - setuptools=="$STOOLS_VER" + python -m pip install --upgrade pip=="$PIP_VER" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" if [[ "with-unit-test" =~ ^($options)$ ]]; then - python -m pip install -U -r python/requirements_test.txt + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt" fi if [[ "with-cuda" =~ ^($options)$ ]]; then TF_ARCH_NAME=tensorflow @@ -64,7 +61,6 @@ install_python_dependencies() { TORCH_GLNX="torch==${TORCH_VER}+cpu" fi - # TODO: modify other locations to use requirements.txt python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements.txt" if [[ "with-jupyter" =~ ^($options)$ ]]; then python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_jupyter_build.txt" @@ -88,7 +84,6 @@ install_python_dependencies() { fi fi if [ "$BUILD_TENSORFLOW_OPS" == "ON" ] || [ "$BUILD_PYTORCH_OPS" == "ON" ]; then - python -m pip install -U yapf=="$YAPF_VER" # Fix Protobuf compatibility issue # https://stackoverflow.com/a/72493690/1255535 # https://github.com/protocolbuffers/protobuf/issues/10051 @@ -241,8 +236,8 @@ test_wheel() { python -m venv open3d_test.venv # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ - setuptools=="$STOOLS_VER" + python -m pip install -U pip=="$PIP_VER" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" echo -n "Using python: $(command -v python)" python --version echo -n "Using pip: " @@ -285,7 +280,7 @@ test_wheel() { run_python_tests() { # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install -U -r python/requirements_test.txt + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt" echo Add --randomly-seed=SEED to the test command to reproduce test order. pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/) if [ "$BUILD_PYTORCH_OPS" == "OFF" ] && [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then @@ -356,9 +351,8 @@ install_docs_dependencies() { echo Install Python dependencies for building docs command -v python python -V - python -m pip install -U -q "wheel==$WHEEL_VER" \ - "pip==$PIP_VER" - python -m pip install -U -q "yapf==$YAPF_VER" + python -m pip install -U -q "pip==$PIP_VER" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" if [[ -d "$1" ]]; then OPEN3D_ML_ROOT="$1" echo Installing Open3D-ML dependencies from "${OPEN3D_ML_ROOT}"