doc(pip_parse_vendored): fix the example (#1942)

With this change we are now again generating a working
`requirements.bzl` file which broke during our switch
to using the `hub` repo.

Fixes #1918.

Author: aignas

CHANGELOG.md

@@ -80,6 +80,8 @@ A brief description of the categories of changes:
   autodetection would match a different toolchain than expected. This may yield
   to toolchain selection failures when the python toolchain is not registered,
   but is requested via `//python/config_settings:python_version` flag setting.
+* (doc) Fix the `WORKSPACE` requirement vendoring example. Fixes
+  [#1918](https://github.com/bazelbuild/rules_python/issues/1918).
 
 ### Added
 * (rules) Precompiling Python source at build time is available. but is

examples/pip_parse_vendored/BUILD.bazel

@@ -1,6 +1,9 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
+load("@rules_python//python:defs.bzl", "py_test")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("//:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements", "requirement")
 
 # This rule adds a convenient way to update the requirements.txt
 # lockfile based on the requirements.in.
@@ -9,6 +12,29 @@ compile_pip_requirements(
     src = "requirements.in",
 )
 
+# The requirements.bzl file is using the hub repo to access packages via the
+# `requirement` macro and when the requirements.bzl is vendored, the hub
+# repo won't be present. As a result, we have to adjust the label scheme in
+# the requirements.bzl to make sure that they continue to work.
+genrule(
+    name = "requirement_bzl",
+    srcs = ["@pip_deps_to_be_vendored//:requirements.bzl"],
+    outs = ["requirements.clean.bzl"],
+    cmd = " | ".join([
+        "cat $<",
+        # Substitute the name of the hub to ensure that the dependencies do
+        # not require the hub repo initialized in the WORKSPACE.
+        "sed -e 's/pip_deps_to_be_vendored/my_project_pip_deps_vendored/g'",
+        # Change the labels from using the hub repo to using the spoke repos
+        # directly.
+        "sed -e 's|//\\([^:]*\\):pkg|_\\1//:pkg|g'",
+        "sed -e 's|//\\([^:]*\\):whl|_\\1//:whl|g'",
+        "sed -e 's|//\\([^:]*\\):data|_\\1//:data|g'",
+        # Change the convenience macros to use the same naming.
+        "sed -e 's|//{}:{}|_{}//:{}|g' >$@",
+    ]),
+)
+
 write_file(
     name = "gen_update",
     out = "update.sh",
@@ -17,14 +43,14 @@ write_file(
         "#!/usr/bin/env bash",
         # Bazel gives us a way to access the source folder!
         "cd $BUILD_WORKSPACE_DIRECTORY",
-        "cp -fv bazel-pip_parse_vendored/external/pip/requirements.bzl requirements.bzl",
+        "cp -fv bazel-bin/requirements.clean.bzl requirements.bzl",
     ],
 )
 
 sh_binary(
     name = "vendor_requirements",
     srcs = ["update.sh"],
-    data = ["@pip//:requirements.bzl"],
+    data = [":requirement_bzl"],
 )
 
 # Similarly ensures that the requirements.bzl file is updated
@@ -33,5 +59,18 @@ diff_test(
     name = "test_vendored",
     failure_message = "Please run:  bazel run //:vendor_requirements",
     file1 = "requirements.bzl",
-    file2 = "@pip//:requirements.bzl",
+    file2 = "requirement_bzl",
+)
+
+py_test(
+    name = "test_dependency_usage",
+    srcs = ["test_dependency_usage.py"],
+    deps = [
+        requirement("requests"),
+    ],
+)
+
+build_test(
+    name = "test_requirement_lists",
+    targets = all_requirements + all_whl_requirements + all_data_requirements,
 )

examples/pip_parse_vendored/WORKSPACE

@@ -1,4 +1,4 @@
-workspace(name = "pip_repository_annotations_example")
+workspace(name = "pip_parse_vendored_example")
 
 local_repository(
     name = "rules_python",
@@ -23,7 +23,8 @@ load("@rules_python//python:pip.bzl", "pip_parse")
 # verify by inspection; the environment variables at a later time, when we download the
 # packages, will be the ones that take effect.
 pip_parse(
-    name = "pip",
+    # We choose a unique name here to make sure we can do some cleanup on it.
+    name = "pip_deps_to_be_vendored",
     envsubst = ["PIP_RETRIES"],
     extra_pip_args = ["--retries=${PIP_RETRIES:-5}"],
     python_interpreter_target = "@python39_host//:python",

examples/pip_parse_vendored/requirements.bzl

@@ -6,34 +6,34 @@
 load("@rules_python//python:pip.bzl", "pip_utils")
 load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")
 
-all_requirements = ["@pip//certifi:pkg", "@pip//charset_normalizer:pkg", "@pip//idna:pkg", "@pip//requests:pkg", "@pip//urllib3:pkg"]
+all_requirements = ["@my_project_pip_deps_vendored_certifi//:pkg", "@my_project_pip_deps_vendored_charset_normalizer//:pkg", "@my_project_pip_deps_vendored_idna//:pkg", "@my_project_pip_deps_vendored_requests//:pkg", "@my_project_pip_deps_vendored_urllib3//:pkg"]
 
-all_whl_requirements_by_package = {"certifi": "@pip//certifi:whl", "charset_normalizer": "@pip//charset_normalizer:whl", "idna": "@pip//idna:whl", "requests": "@pip//requests:whl", "urllib3": "@pip//urllib3:whl"}
+all_whl_requirements_by_package = {"certifi": "@my_project_pip_deps_vendored_certifi//:whl", "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl", "idna": "@my_project_pip_deps_vendored_idna//:whl", "requests": "@my_project_pip_deps_vendored_requests//:whl", "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl"}
 
 all_whl_requirements = all_whl_requirements_by_package.values()
 
-all_data_requirements = ["@pip//certifi:data", "@pip//charset_normalizer:data", "@pip//idna:data", "@pip//requests:data", "@pip//urllib3:data"]
+all_data_requirements = ["@my_project_pip_deps_vendored_certifi//:data", "@my_project_pip_deps_vendored_charset_normalizer//:data", "@my_project_pip_deps_vendored_idna//:data", "@my_project_pip_deps_vendored_requests//:data", "@my_project_pip_deps_vendored_urllib3//:data"]
 
-_packages = [("pip_certifi", "certifi==2023.7.22     --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082     --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("pip_charset_normalizer", "charset-normalizer==2.1.1     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13     --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc     --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
-_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
+_packages = [("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22     --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082     --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("my_project_pip_deps_vendored_idna", "idna==3.4     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("my_project_pip_deps_vendored_requests", "requests==2.28.1     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13     --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc     --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
+_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "my_project_pip_deps_vendored", "repo_prefix": "my_project_pip_deps_vendored_", "timeout": 600}
 _annotations = {}
 
 def requirement(name):
-    return "@pip//{}:{}".format(pip_utils.normalize_name(name), "pkg")
+    return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "pkg")
 
 def whl_requirement(name):
-    return "@pip//{}:{}".format(pip_utils.normalize_name(name), "whl")
+    return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "whl")
 
 def data_requirement(name):
-    return "@pip//{}:{}".format(pip_utils.normalize_name(name), "data")
+    return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "data")
 
 def dist_info_requirement(name):
-    return "@pip//{}:{}".format(pip_utils.normalize_name(name), "dist_info")
+    return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "dist_info")
 
 def entry_point(pkg, script = None):
     if not script:
         script = pkg
-    return "@pip_" + pip_utils.normalize_name(pkg) + "//:rules_python_wheel_entry_point_" + script
+    return "@my_project_pip_deps_vendored_" + pip_utils.normalize_name(pkg) + "//:rules_python_wheel_entry_point_" + script
 
 def _get_annotation(requirement):
     # This expects to parse `setuptools==58.2.0     --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11`
@@ -58,10 +58,10 @@ def install_deps(**whl_library_kwargs):
         for requirement in group_requirements
     }
 
-    group_repo = "pip__groups"
+    group_repo = "my_project_pip_deps_vendored__groups"
     group_library(
         name = group_repo,
-        repo_prefix = "pip_",
+        repo_prefix = "my_project_pip_deps_vendored_",
         groups = all_requirement_groups,
     )
 
@@ -70,7 +70,7 @@ def install_deps(**whl_library_kwargs):
     whl_config.update(whl_library_kwargs)
 
     for name, requirement in _packages:
-        group_name = requirement_group_mapping.get(name.replace("pip_", ""))
+        group_name = requirement_group_mapping.get(name.replace("my_project_pip_deps_vendored_", ""))
         group_deps = all_requirement_groups.get(group_name, [])
 
         whl_library(

examples/pip_parse_vendored/test_dependency_usage.py

@@ -0,0 +1,12 @@
+import unittest
+
+import requests
+
+
+class TestDependencies(unittest.TestCase):
+    def test_import(self):
+        self.assertIsNotNone(requests.get)
+
+
+if __name__ == "__main__":
+    unittest.main()