From 8399bc5e7fd14a8a379ceade99712c6f6e979486 Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:19:52 +0900 Subject: [PATCH] doc(pip_parse_vendored): fix the example With this change we are now again generating a working `requirements.bzl` file which broke during our switch to using the `hub` repo. Fixes #1918. --- CHANGELOG.md | 2 + examples/pip_parse_vendored/BUILD.bazel | 45 +++++++++++++++++-- examples/pip_parse_vendored/WORKSPACE | 5 ++- examples/pip_parse_vendored/requirements.bzl | 26 +++++------ .../test_dependency_usage.py | 12 +++++ 5 files changed, 72 insertions(+), 18 deletions(-) create mode 100644 examples/pip_parse_vendored/test_dependency_usage.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0720a36a59..3a7c990664 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,8 @@ A brief description of the categories of changes: the `experimental_index_url` feature which will fetch metadata from PyPI or a different private index and write the contents to the lock file. Fixes [#1643](https://github.com/bazelbuild/rules_python/issues/1643). +* (doc) Fix the `WORKSPACE` requirement vendoring example. Fixes + [#1918](https://github.com/bazelbuild/rules_python/issues/1918). ### Added * (rules) Precompiling Python source at build time is available. but is diff --git a/examples/pip_parse_vendored/BUILD.bazel b/examples/pip_parse_vendored/BUILD.bazel index b856c94525..e2b1f5d49b 100644 --- a/examples/pip_parse_vendored/BUILD.bazel +++ b/examples/pip_parse_vendored/BUILD.bazel @@ -1,6 +1,9 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("@rules_python//python:defs.bzl", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") +load("//:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements", "requirement") # This rule adds a convenient way to update the requirements.txt # lockfile based on the requirements.in. @@ -9,6 +12,29 @@ compile_pip_requirements( src = "requirements.in", ) +# The requirements.bzl file is using the hub repo to access packages via the +# `requirement` macro and when the requirements.bzl is vendored, the hub +# repo won't be present. As a result, we have to adjust the label scheme in +# the requirements.bzl to make sure that they continue to work. +genrule( + name = "requirement_bzl", + srcs = ["@pip_deps_to_be_vendored//:requirements.bzl"], + outs = ["requirements.clean.bzl"], + cmd = " | ".join([ + "cat $<", + # Substitute the name of the hub to ensure that the dependencies do + # not require the hub repo initialized in the WORKSPACE. + "sed -e 's/pip_deps_to_be_vendored/my_project_pip_deps_vendored/g'", + # Change the labels from using the hub repo to using the spoke repos + # directly. + "sed -e 's|//\\([^:]*\\):pkg|_\\1//:pkg|g'", + "sed -e 's|//\\([^:]*\\):whl|_\\1//:whl|g'", + "sed -e 's|//\\([^:]*\\):data|_\\1//:data|g'", + # Change the convenience macros to use the same naming. + "sed -e 's|//{}:{}|_{}//:{}|g' >$@", + ]), +) + write_file( name = "gen_update", out = "update.sh", @@ -17,14 +43,14 @@ write_file( "#!/usr/bin/env bash", # Bazel gives us a way to access the source folder! "cd $BUILD_WORKSPACE_DIRECTORY", - "cp -fv bazel-pip_parse_vendored/external/pip/requirements.bzl requirements.bzl", + "cp -fv bazel-bin/requirements.clean.bzl requirements.bzl", ], ) sh_binary( name = "vendor_requirements", srcs = ["update.sh"], - data = ["@pip//:requirements.bzl"], + data = [":requirement_bzl"], ) # Similarly ensures that the requirements.bzl file is updated @@ -33,5 +59,18 @@ diff_test( name = "test_vendored", failure_message = "Please run: bazel run //:vendor_requirements", file1 = "requirements.bzl", - file2 = "@pip//:requirements.bzl", + file2 = "requirement_bzl", +) + +py_test( + name = "test_dependency_usage", + srcs = ["test_dependency_usage.py"], + deps = [ + requirement("requests"), + ], +) + +build_test( + name = "test_requirement_lists", + targets = all_requirements + all_whl_requirements + all_data_requirements, ) diff --git a/examples/pip_parse_vendored/WORKSPACE b/examples/pip_parse_vendored/WORKSPACE index 7904724d9e..d7a11ea596 100644 --- a/examples/pip_parse_vendored/WORKSPACE +++ b/examples/pip_parse_vendored/WORKSPACE @@ -1,4 +1,4 @@ -workspace(name = "pip_repository_annotations_example") +workspace(name = "pip_parse_vendored_example") local_repository( name = "rules_python", @@ -23,7 +23,8 @@ load("@rules_python//python:pip.bzl", "pip_parse") # verify by inspection; the environment variables at a later time, when we download the # packages, will be the ones that take effect. pip_parse( - name = "pip", + # We choose a unique name here to make sure we can do some cleanup on it. + name = "pip_deps_to_be_vendored", envsubst = ["PIP_RETRIES"], extra_pip_args = ["--retries=${PIP_RETRIES:-5}"], python_interpreter_target = "@python39_host//:python", diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 5c2391bd4c..c6465955ba 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -6,34 +6,34 @@ load("@rules_python//python:pip.bzl", "pip_utils") load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library") -all_requirements = ["@pip//certifi:pkg", "@pip//charset_normalizer:pkg", "@pip//idna:pkg", "@pip//requests:pkg", "@pip//urllib3:pkg"] +all_requirements = ["@my_project_pip_deps_vendored_certifi//:pkg", "@my_project_pip_deps_vendored_charset_normalizer//:pkg", "@my_project_pip_deps_vendored_idna//:pkg", "@my_project_pip_deps_vendored_requests//:pkg", "@my_project_pip_deps_vendored_urllib3//:pkg"] -all_whl_requirements_by_package = {"certifi": "@pip//certifi:whl", "charset_normalizer": "@pip//charset_normalizer:whl", "idna": "@pip//idna:whl", "requests": "@pip//requests:whl", "urllib3": "@pip//urllib3:whl"} +all_whl_requirements_by_package = {"certifi": "@my_project_pip_deps_vendored_certifi//:whl", "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl", "idna": "@my_project_pip_deps_vendored_idna//:whl", "requests": "@my_project_pip_deps_vendored_requests//:whl", "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl"} all_whl_requirements = all_whl_requirements_by_package.values() -all_data_requirements = ["@pip//certifi:data", "@pip//charset_normalizer:data", "@pip//idna:data", "@pip//requests:data", "@pip//urllib3:data"] +all_data_requirements = ["@my_project_pip_deps_vendored_certifi//:data", "@my_project_pip_deps_vendored_charset_normalizer//:data", "@my_project_pip_deps_vendored_idna//:data", "@my_project_pip_deps_vendored_requests//:data", "@my_project_pip_deps_vendored_urllib3//:data"] -_packages = [("pip_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("pip_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")] -_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} +_packages = [("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("my_project_pip_deps_vendored_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("my_project_pip_deps_vendored_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")] +_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "my_project_pip_deps_vendored", "repo_prefix": "my_project_pip_deps_vendored_", "timeout": 600} _annotations = {} def requirement(name): - return "@pip//{}:{}".format(pip_utils.normalize_name(name), "pkg") + return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "pkg") def whl_requirement(name): - return "@pip//{}:{}".format(pip_utils.normalize_name(name), "whl") + return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "whl") def data_requirement(name): - return "@pip//{}:{}".format(pip_utils.normalize_name(name), "data") + return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "data") def dist_info_requirement(name): - return "@pip//{}:{}".format(pip_utils.normalize_name(name), "dist_info") + return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "dist_info") def entry_point(pkg, script = None): if not script: script = pkg - return "@pip_" + pip_utils.normalize_name(pkg) + "//:rules_python_wheel_entry_point_" + script + return "@my_project_pip_deps_vendored_" + pip_utils.normalize_name(pkg) + "//:rules_python_wheel_entry_point_" + script def _get_annotation(requirement): # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` @@ -58,10 +58,10 @@ def install_deps(**whl_library_kwargs): for requirement in group_requirements } - group_repo = "pip__groups" + group_repo = "my_project_pip_deps_vendored__groups" group_library( name = group_repo, - repo_prefix = "pip_", + repo_prefix = "my_project_pip_deps_vendored_", groups = all_requirement_groups, ) @@ -70,7 +70,7 @@ def install_deps(**whl_library_kwargs): whl_config.update(whl_library_kwargs) for name, requirement in _packages: - group_name = requirement_group_mapping.get(name.replace("pip_", "")) + group_name = requirement_group_mapping.get(name.replace("my_project_pip_deps_vendored_", "")) group_deps = all_requirement_groups.get(group_name, []) whl_library( diff --git a/examples/pip_parse_vendored/test_dependency_usage.py b/examples/pip_parse_vendored/test_dependency_usage.py new file mode 100644 index 0000000000..e2cf970d9d --- /dev/null +++ b/examples/pip_parse_vendored/test_dependency_usage.py @@ -0,0 +1,12 @@ +import unittest + +import requests + + +class TestDependencies(unittest.TestCase): + def test_import(self): + self.assertIsNotNone(requests.get) + + +if __name__ == "__main__": + unittest.main()