From 867f8b0b9425b7de88171f070bc29b9b23ba9562 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 13:33:33 +0900
Subject: [PATCH 01/25] chore: add aignas to CODEOWNERS
---
.github/CODEOWNERS | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 3449bcfac0..4f0ca4d0a8 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -11,5 +11,5 @@
/python/private/toolchains_repo.bzl @f0rmiga
/python/tests/toolchains/ @f0rmiga
-# pip_parse related code
-/python/pip_install/ @hrfuller
+# PyPI integration related code
+/python/private/pypi/ @aignas
From 79fa303ed74462d1dd932bace897eb9e63953d23 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 13:33:58 +0900
Subject: [PATCH 02/25] chore: add a new directory for storing PyPI related
code
---
python/private/pypi/BUILD.bazel | 0
python/private/pypi/README.md | 9 +++++++++
2 files changed, 9 insertions(+)
create mode 100644 python/private/pypi/BUILD.bazel
create mode 100644 python/private/pypi/README.md
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/python/private/pypi/README.md b/python/private/pypi/README.md
new file mode 100644
index 0000000000..6be5703912
--- /dev/null
+++ b/python/private/pypi/README.md
@@ -0,0 +1,9 @@
+# PyPI integration code
+
+This code is for integrating with PyPI and other compatible indexes. At the
+moment we have code for:
+* Downloading packages using `pip` or `repository_ctx.download`.
+* Interacting with PyPI compatible indexes via [SimpleAPI] spec.
+* Locking a `requirements.in` or [PEP621] compliant `pyproject.toml`.
+
+[PEP621]: https://peps.python.org/pep-0621/
From b5afdadc95da7eca93d99a92757823a70d5a0363 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 13:43:09 +0900
Subject: [PATCH 03/25] move pypi_index_sources.bzl to private/pypi
---
python/private/BUILD.bazel | 8 +---
python/private/parse_requirements.bzl | 4 +-
python/private/pypi/BUILD.bazel | 37 +++++++++++++++++++
.../index_sources.bzl} | 2 +-
tests/private/pypi_index_sources/BUILD.bazel | 3 --
tests/pypi/index_sources/BUILD.bazel | 3 ++
.../index_sources/index_sources_tests.bzl} | 8 ++--
7 files changed, 49 insertions(+), 16 deletions(-)
rename python/private/{pypi_index_sources.bzl => pypi/index_sources.bzl} (98%)
delete mode 100644 tests/private/pypi_index_sources/BUILD.bazel
create mode 100644 tests/pypi/index_sources/BUILD.bazel
rename tests/{private/pypi_index_sources/pypi_index_sources_tests.bzl => pypi/index_sources/index_sources_tests.bzl} (88%)
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index cd385e3700..0164321d41 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -31,6 +31,7 @@ filegroup(
"//python/private/bzlmod:distribution",
"//python/private/common:distribution",
"//python/private/proto:distribution",
+ "//python/private/pypi:distribution",
"//python/private/whl_filegroup:distribution",
"//tools/build_defs/python/private:distribution",
],
@@ -136,9 +137,9 @@ bzl_library(
srcs = ["parse_requirements.bzl"],
deps = [
":normalize_name_bzl",
- ":pypi_index_sources_bzl",
":whl_target_platforms_bzl",
"//python/pip_install:requirements_parser_bzl",
+ "//python/private/pypi:index_sources_bzl",
],
)
@@ -176,11 +177,6 @@ bzl_library(
],
)
-bzl_library(
- name = "pypi_index_sources_bzl",
- srcs = ["pypi_index_sources.bzl"],
-)
-
bzl_library(
name = "py_cc_toolchain_bzl",
srcs = [
diff --git a/python/private/parse_requirements.bzl b/python/private/parse_requirements.bzl
index 21e132b4f8..dcc2442199 100644
--- a/python/private/parse_requirements.bzl
+++ b/python/private/parse_requirements.bzl
@@ -27,8 +27,8 @@ behavior.
"""
load("//python/pip_install:requirements_parser.bzl", "parse")
+load("//python/private/pypi:index_sources.bzl", "index_sources")
load(":normalize_name.bzl", "normalize_name")
-load(":pypi_index_sources.bzl", "get_simpleapi_sources")
load(":whl_target_platforms.bzl", "select_whls", "whl_target_platforms")
# This includes the vendored _translate_cpu and _translate_os from
@@ -317,7 +317,7 @@ def parse_requirements(
(requirement_line, ",".join(extra_pip_args)),
struct(
distribution = distribution,
- srcs = get_simpleapi_sources(requirement_line),
+ srcs = index_sources(requirement_line),
requirement_line = requirement_line,
target_platforms = [],
extra_pip_args = extra_pip_args,
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index e69de29bb2..3c14ec2e61 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -0,0 +1,37 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+package(default_visibility = ["//:__subpackages__"])
+
+licenses(["notice"])
+
+filegroup(
+ name = "distribution",
+ srcs = glob(["**"]),
+ visibility = ["//python/private:__pkg__"],
+)
+
+# Filegroup of bzl files that can be used by downstream rules for documentation generation
+filegroup(
+ name = "bzl",
+ srcs = glob(["**/*.bzl"]),
+ visibility = ["//python/private:__pkg__"],
+)
+
+bzl_library(
+ name = "index_sources_bzl",
+ srcs = ["index_sources.bzl"],
+)
diff --git a/python/private/pypi_index_sources.bzl b/python/private/pypi/index_sources.bzl
similarity index 98%
rename from python/private/pypi_index_sources.bzl
rename to python/private/pypi/index_sources.bzl
index 470a8c9f5a..21660141db 100644
--- a/python/private/pypi_index_sources.bzl
+++ b/python/private/pypi/index_sources.bzl
@@ -16,7 +16,7 @@
A file that houses private functions used in the `bzlmod` extension with the same name.
"""
-def get_simpleapi_sources(line):
+def index_sources(line):
"""Get PyPI sources from a requirements.txt line.
We interpret the spec described in
diff --git a/tests/private/pypi_index_sources/BUILD.bazel b/tests/private/pypi_index_sources/BUILD.bazel
deleted file mode 100644
index 212615f480..0000000000
--- a/tests/private/pypi_index_sources/BUILD.bazel
+++ /dev/null
@@ -1,3 +0,0 @@
-load(":pypi_index_sources_tests.bzl", "pypi_index_sources_test_suite")
-
-pypi_index_sources_test_suite(name = "pypi_index_sources_tests")
diff --git a/tests/pypi/index_sources/BUILD.bazel b/tests/pypi/index_sources/BUILD.bazel
new file mode 100644
index 0000000000..7cd327abef
--- /dev/null
+++ b/tests/pypi/index_sources/BUILD.bazel
@@ -0,0 +1,3 @@
+load(":index_sources_tests.bzl", "index_sources_test_suite")
+
+index_sources_test_suite(name = "index_sources_tests")
diff --git a/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl b/tests/pypi/index_sources/index_sources_tests.bzl
similarity index 88%
rename from tests/private/pypi_index_sources/pypi_index_sources_tests.bzl
rename to tests/pypi/index_sources/index_sources_tests.bzl
index 48d790fc68..0a767078ba 100644
--- a/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl
+++ b/tests/pypi/index_sources/index_sources_tests.bzl
@@ -15,7 +15,7 @@
""
load("@rules_testing//lib:test_suite.bzl", "test_suite")
-load("//python/private:pypi_index_sources.bzl", "get_simpleapi_sources") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:index_sources.bzl", "index_sources") # buildifier: disable=bzl-visibility
_tests = []
@@ -27,7 +27,7 @@ def _test_no_simple_api_sources(env):
"foo==0.0.1 @ https://someurl.org; python_version < 2.7 --hash=sha256:deadbeef",
]
for input in inputs:
- got = get_simpleapi_sources(input)
+ got = index_sources(input)
env.expect.that_collection(got.shas).contains_exactly([])
env.expect.that_str(got.version).equals("0.0.1")
@@ -45,13 +45,13 @@ def _test_simple_api_sources(env):
],
}
for input, want_shas in tests.items():
- got = get_simpleapi_sources(input)
+ got = index_sources(input)
env.expect.that_collection(got.shas).contains_exactly(want_shas)
env.expect.that_str(got.version).equals("0.0.2")
_tests.append(_test_simple_api_sources)
-def pypi_index_sources_test_suite(name):
+def index_sources_test_suite(name):
"""Create the test suite.
Args:
From b0d57b6849f9e0659525d29a3d3e106048ee30a5 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 13:57:30 +0900
Subject: [PATCH 04/25] chore: move parse_requirements_txt to private/pypi
---
python/pip_install/BUILD.bazel | 5 -
python/pip_install/requirements_parser.bzl | 118 +---------------
python/private/BUILD.bazel | 3 +-
python/private/parse_requirements.bzl | 4 +-
python/private/pypi/BUILD.bazel | 5 +
.../private/pypi/parse_requirements_txt.bzl | 133 ++++++++++++++++++
.../requirements_parser/BUILD.bazel | 3 -
tests/pypi/parse_requirements_txt/BUILD.bazel | 3 +
.../parse_requirements_txt_tests.bzl} | 54 +++----
9 files changed, 173 insertions(+), 155 deletions(-)
create mode 100644 python/private/pypi/parse_requirements_txt.bzl
delete mode 100644 tests/pip_install/requirements_parser/BUILD.bazel
create mode 100644 tests/pypi/parse_requirements_txt/BUILD.bazel
rename tests/{pip_install/requirements_parser/requirements_parser_tests.bzl => pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl} (88%)
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 91f2ec7b59..961027f706 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -51,11 +51,6 @@ bzl_library(
],
)
-bzl_library(
- name = "requirements_parser_bzl",
- srcs = ["requirements_parser.bzl"],
-)
-
bzl_library(
name = "repositories_bzl",
srcs = ["repositories.bzl"],
diff --git a/python/pip_install/requirements_parser.bzl b/python/pip_install/requirements_parser.bzl
index 3b49fdf181..82ec1b946c 100644
--- a/python/pip_install/requirements_parser.bzl
+++ b/python/pip_install/requirements_parser.bzl
@@ -14,120 +14,6 @@
"Pip requirements parser for Starlark"
-_STATE = struct(
- # Consume extraneous whitespace
- ConsumeSpace = 0,
- # Consume a comment
- ConsumeComment = 1,
- # Parse the name of a pip package
- ParseDependency = 2,
- # Parse a full requirement line
- ParseRequirement = 3,
- # Parse a pip option
- ParseOption = 4,
-)
+load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt")
-EOF = {}
-
-def parse(content):
- """A simplistic (and incomplete) pip requirements lockfile parser.
-
- Parses package names and their full requirement lines, as well pip
- options.
-
- Args:
- content: lockfile content as a string
-
- Returns:
- Struct with fields `requirements` and `options`.
-
- requirements: List of requirements, where each requirement is a 2-element
- tuple containing the package name and the requirement line.
- E.g., [(certifi', 'certifi==2021.10.8 --hash=sha256:7888...'), ...]
-
- options: List of pip option lines
- """
- content = content.replace("\r", "")
-
- result = struct(
- requirements = [],
- options = [],
- )
- state = _STATE.ConsumeSpace
- buffer = ""
-
- inputs = content.elems()[:]
- inputs.append(EOF)
-
- for input in inputs:
- if state == _STATE.ConsumeSpace:
- (state, buffer) = _handleConsumeSpace(input)
- elif state == _STATE.ConsumeComment:
- (state, buffer) = _handleConsumeComment(input, buffer, result)
- elif state == _STATE.ParseDependency:
- (state, buffer) = _handleParseDependency(input, buffer, result)
- elif state == _STATE.ParseOption:
- (state, buffer) = _handleParseOption(input, buffer, result)
- elif state == _STATE.ParseRequirement:
- (state, buffer) = _handleParseRequirement(input, buffer, result)
- else:
- fail("Unknown state %d" % state)
-
- return result
-
-def _handleConsumeSpace(input):
- if input == EOF:
- return (_STATE.ConsumeSpace, "")
- if input.isspace():
- return (_STATE.ConsumeSpace, "")
- elif input == "#":
- return (_STATE.ConsumeComment, "")
- elif input == "-":
- return (_STATE.ParseOption, input)
-
- return (_STATE.ParseDependency, input)
-
-def _handleConsumeComment(input, buffer, result):
- if input == "\n":
- if len(result.requirements) > 0 and len(result.requirements[-1]) == 1:
- result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n"))
- return (_STATE.ConsumeSpace, "")
- elif len(buffer) > 0:
- result.options.append(buffer.rstrip(" \n"))
- return (_STATE.ConsumeSpace, "")
- return (_STATE.ConsumeSpace, "")
- return (_STATE.ConsumeComment, buffer)
-
-def _handleParseDependency(input, buffer, result):
- if input == EOF:
- fail("Enountered unexpected end of file while parsing requirement")
- elif input.isspace() or input in [">", "<", "~", "=", ";", "["]:
- result.requirements.append((buffer,))
- return (_STATE.ParseRequirement, buffer + input)
-
- return (_STATE.ParseDependency, buffer + input)
-
-def _handleParseOption(input, buffer, result):
- if input == "\n" and buffer.endswith("\\"):
- return (_STATE.ParseOption, buffer[0:-1])
- elif input == " ":
- result.options.append(buffer.rstrip("\n"))
- return (_STATE.ParseOption, "")
- elif input == "\n" or input == EOF:
- result.options.append(buffer.rstrip("\n"))
- return (_STATE.ConsumeSpace, "")
- elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()):
- return (_STATE.ConsumeComment, buffer)
-
- return (_STATE.ParseOption, buffer + input)
-
-def _handleParseRequirement(input, buffer, result):
- if input == "\n" and buffer.endswith("\\"):
- return (_STATE.ParseRequirement, buffer[0:-1])
- elif input == "\n" or input == EOF:
- result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n"))
- return (_STATE.ConsumeSpace, "")
- elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()):
- return (_STATE.ConsumeComment, buffer)
-
- return (_STATE.ParseRequirement, buffer + input)
+parse = parse_requirements_txt
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 0164321d41..1524e42acd 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -138,8 +138,8 @@ bzl_library(
deps = [
":normalize_name_bzl",
":whl_target_platforms_bzl",
- "//python/pip_install:requirements_parser_bzl",
"//python/private/pypi:index_sources_bzl",
+ "//python/private/pypi:parse_requirements_txt_bzl",
],
)
@@ -172,7 +172,6 @@ bzl_library(
":auth_bzl",
":normalize_name_bzl",
":text_util_bzl",
- "//python/pip_install:requirements_parser_bzl",
"//python/private/bzlmod:bazel_features_bzl",
],
)
diff --git a/python/private/parse_requirements.bzl b/python/private/parse_requirements.bzl
index dcc2442199..1b14ae0dc1 100644
--- a/python/private/parse_requirements.bzl
+++ b/python/private/parse_requirements.bzl
@@ -26,8 +26,8 @@ file for the host platform to be backwards compatible with the legacy
behavior.
"""
-load("//python/pip_install:requirements_parser.bzl", "parse")
load("//python/private/pypi:index_sources.bzl", "index_sources")
+load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt")
load(":normalize_name.bzl", "normalize_name")
load(":whl_target_platforms.bzl", "select_whls", "whl_target_platforms")
@@ -271,7 +271,7 @@ def parse_requirements(
# Parse the requirements file directly in starlark to get the information
# needed for the whl_library declarations later.
- parse_result = parse(contents)
+ parse_result = parse_requirements_txt(contents)
# Replicate a surprising behavior that WORKSPACE builds allowed:
# Defining a repo with the same name multiple times, but only the last
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 3c14ec2e61..4d9a7164e9 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -35,3 +35,8 @@ bzl_library(
name = "index_sources_bzl",
srcs = ["index_sources.bzl"],
)
+
+bzl_library(
+ name = "parse_requirements_txt_bzl",
+ srcs = ["parse_requirements_txt.bzl"],
+)
diff --git a/python/private/pypi/parse_requirements_txt.bzl b/python/private/pypi/parse_requirements_txt.bzl
new file mode 100644
index 0000000000..6f51d034da
--- /dev/null
+++ b/python/private/pypi/parse_requirements_txt.bzl
@@ -0,0 +1,133 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Pip requirements parser for Starlark."""
+
+_STATE = struct(
+ # Consume extraneous whitespace
+ ConsumeSpace = 0,
+ # Consume a comment
+ ConsumeComment = 1,
+ # Parse the name of a pip package
+ ParseDependency = 2,
+ # Parse a full requirement line
+ ParseRequirement = 3,
+ # Parse a pip option
+ ParseOption = 4,
+)
+
+EOF = {}
+
+def parse_requirements_txt(content):
+ """A simplistic (and incomplete) pip requirements lockfile parser.
+
+ Parses package names and their full requirement lines, as well pip
+ options.
+
+ Args:
+ content: lockfile content as a string
+
+ Returns:
+ Struct with fields `requirements` and `options`.
+
+ requirements: List of requirements, where each requirement is a 2-element
+ tuple containing the package name and the requirement line.
+ E.g., [(certifi', 'certifi==2021.10.8 --hash=sha256:7888...'), ...]
+
+ options: List of pip option lines
+ """
+ content = content.replace("\r", "")
+
+ result = struct(
+ requirements = [],
+ options = [],
+ )
+ state = _STATE.ConsumeSpace
+ buffer = ""
+
+ inputs = content.elems()[:]
+ inputs.append(EOF)
+
+ for input in inputs:
+ if state == _STATE.ConsumeSpace:
+ (state, buffer) = _handleConsumeSpace(input)
+ elif state == _STATE.ConsumeComment:
+ (state, buffer) = _handleConsumeComment(input, buffer, result)
+ elif state == _STATE.ParseDependency:
+ (state, buffer) = _handleParseDependency(input, buffer, result)
+ elif state == _STATE.ParseOption:
+ (state, buffer) = _handleParseOption(input, buffer, result)
+ elif state == _STATE.ParseRequirement:
+ (state, buffer) = _handleParseRequirement(input, buffer, result)
+ else:
+ fail("Unknown state %d" % state)
+
+ return result
+
+def _handleConsumeSpace(input):
+ if input == EOF:
+ return (_STATE.ConsumeSpace, "")
+ if input.isspace():
+ return (_STATE.ConsumeSpace, "")
+ elif input == "#":
+ return (_STATE.ConsumeComment, "")
+ elif input == "-":
+ return (_STATE.ParseOption, input)
+
+ return (_STATE.ParseDependency, input)
+
+def _handleConsumeComment(input, buffer, result):
+ if input == "\n":
+ if len(result.requirements) > 0 and len(result.requirements[-1]) == 1:
+ result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n"))
+ return (_STATE.ConsumeSpace, "")
+ elif len(buffer) > 0:
+ result.options.append(buffer.rstrip(" \n"))
+ return (_STATE.ConsumeSpace, "")
+ return (_STATE.ConsumeSpace, "")
+ return (_STATE.ConsumeComment, buffer)
+
+def _handleParseDependency(input, buffer, result):
+ if input == EOF:
+ fail("Enountered unexpected end of file while parsing requirement")
+ elif input.isspace() or input in [">", "<", "~", "=", ";", "["]:
+ result.requirements.append((buffer,))
+ return (_STATE.ParseRequirement, buffer + input)
+
+ return (_STATE.ParseDependency, buffer + input)
+
+def _handleParseOption(input, buffer, result):
+ if input == "\n" and buffer.endswith("\\"):
+ return (_STATE.ParseOption, buffer[0:-1])
+ elif input == " ":
+ result.options.append(buffer.rstrip("\n"))
+ return (_STATE.ParseOption, "")
+ elif input == "\n" or input == EOF:
+ result.options.append(buffer.rstrip("\n"))
+ return (_STATE.ConsumeSpace, "")
+ elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()):
+ return (_STATE.ConsumeComment, buffer)
+
+ return (_STATE.ParseOption, buffer + input)
+
+def _handleParseRequirement(input, buffer, result):
+ if input == "\n" and buffer.endswith("\\"):
+ return (_STATE.ParseRequirement, buffer[0:-1])
+ elif input == "\n" or input == EOF:
+ result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n"))
+ return (_STATE.ConsumeSpace, "")
+ elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()):
+ return (_STATE.ConsumeComment, buffer)
+
+ return (_STATE.ParseRequirement, buffer + input)
diff --git a/tests/pip_install/requirements_parser/BUILD.bazel b/tests/pip_install/requirements_parser/BUILD.bazel
deleted file mode 100644
index 2787f16f94..0000000000
--- a/tests/pip_install/requirements_parser/BUILD.bazel
+++ /dev/null
@@ -1,3 +0,0 @@
-load(":requirements_parser_tests.bzl", parse_requirements_tests = "parse_tests")
-
-parse_requirements_tests(name = "test_parse_requirements")
diff --git a/tests/pypi/parse_requirements_txt/BUILD.bazel b/tests/pypi/parse_requirements_txt/BUILD.bazel
new file mode 100644
index 0000000000..526fa73d4b
--- /dev/null
+++ b/tests/pypi/parse_requirements_txt/BUILD.bazel
@@ -0,0 +1,3 @@
+load(":parse_requirements_txt_tests.bzl", "parse_requirements_txt_test_suite")
+
+parse_requirements_txt_test_suite(name = "parse_requirements_txt_tests")
diff --git a/tests/pip_install/requirements_parser/requirements_parser_tests.bzl b/tests/pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl
similarity index 88%
rename from tests/pip_install/requirements_parser/requirements_parser_tests.bzl
rename to tests/pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl
index 5ea742e70d..f4e899054a 100644
--- a/tests/pip_install/requirements_parser/requirements_parser_tests.bzl
+++ b/tests/pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl
@@ -15,83 +15,83 @@
"Unit tests for yaml.bzl"
load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest")
-load("//python/pip_install:requirements_parser.bzl", "parse")
+load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt") # buildifier: disable=bzl-visibility
def _parse_basic_test_impl(ctx):
env = unittest.begin(ctx)
# Base cases
- asserts.equals(env, [], parse("").requirements)
- asserts.equals(env, [], parse("\n").requirements)
+ asserts.equals(env, [], parse_requirements_txt("").requirements)
+ asserts.equals(env, [], parse_requirements_txt("\n").requirements)
# Various requirement specifiers (https://pip.pypa.io/en/stable/reference/requirement-specifiers/#requirement-specifiers)
- asserts.equals(env, [("SomeProject", "SomeProject")], parse("SomeProject\n").requirements)
- asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse("SomeProject == 1.3\n").requirements)
- asserts.equals(env, [("SomeProject", "SomeProject >= 1.2, < 2.0")], parse("SomeProject >= 1.2, < 2.0\n").requirements)
- asserts.equals(env, [("SomeProject", "SomeProject[foo, bar]")], parse("SomeProject[foo, bar]\n").requirements)
- asserts.equals(env, [("SomeProject", "SomeProject ~= 1.4.2")], parse("SomeProject ~= 1.4.2\n").requirements)
- asserts.equals(env, [("SomeProject", "SomeProject == 5.4 ; python_version < '3.8'")], parse("SomeProject == 5.4 ; python_version < '3.8'\n").requirements)
- asserts.equals(env, [("SomeProject", "SomeProject ; sys_platform == 'win32'")], parse("SomeProject ; sys_platform == 'win32'\n").requirements)
- asserts.equals(env, [("requests", "requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7")], parse("requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7\n").requirements)
+ asserts.equals(env, [("SomeProject", "SomeProject")], parse_requirements_txt("SomeProject\n").requirements)
+ asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse_requirements_txt("SomeProject == 1.3\n").requirements)
+ asserts.equals(env, [("SomeProject", "SomeProject >= 1.2, < 2.0")], parse_requirements_txt("SomeProject >= 1.2, < 2.0\n").requirements)
+ asserts.equals(env, [("SomeProject", "SomeProject[foo, bar]")], parse_requirements_txt("SomeProject[foo, bar]\n").requirements)
+ asserts.equals(env, [("SomeProject", "SomeProject ~= 1.4.2")], parse_requirements_txt("SomeProject ~= 1.4.2\n").requirements)
+ asserts.equals(env, [("SomeProject", "SomeProject == 5.4 ; python_version < '3.8'")], parse_requirements_txt("SomeProject == 5.4 ; python_version < '3.8'\n").requirements)
+ asserts.equals(env, [("SomeProject", "SomeProject ; sys_platform == 'win32'")], parse_requirements_txt("SomeProject ; sys_platform == 'win32'\n").requirements)
+ asserts.equals(env, [("requests", "requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7")], parse_requirements_txt("requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7\n").requirements)
# Multiple requirements
- asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse("""\
+ asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse_requirements_txt("""\
FooProject==1.0.0
BarProject==2.0.0
""").requirements)
- asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse("""\
+ asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse_requirements_txt("""\
FooProject==1.0.0
BarProject==2.0.0
""").requirements)
# Comments
- asserts.equals(env, [("SomeProject", "SomeProject")], parse("""\
+ asserts.equals(env, [("SomeProject", "SomeProject")], parse_requirements_txt("""\
# This is a comment
SomeProject
""").requirements)
- asserts.equals(env, [("SomeProject", "SomeProject")], parse("""\
+ asserts.equals(env, [("SomeProject", "SomeProject")], parse_requirements_txt("""\
SomeProject
# This is a comment
""").requirements)
- asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse("""\
+ asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse_requirements_txt("""\
SomeProject == 1.3 # This is a comment
""").requirements)
- asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse("""\
+ asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse_requirements_txt("""\
FooProject==1.0.0
# Comment
BarProject==2.0.0 #Comment
""").requirements)
- asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49")], parse("""\
+ asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49")], parse_requirements_txt("""\
requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49
""").requirements)
# Multiline
- asserts.equals(env, [("certifi", "certifi==2021.10.8 --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569")], parse("""\
+ asserts.equals(env, [("certifi", "certifi==2021.10.8 --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569")], parse_requirements_txt("""\
certifi==2021.10.8 \
--hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \
--hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569
# via requests
""").requirements)
- asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 --hash=sha256:eca58eb564b134e4ff521a02aa6f566c653835753e1fc8a50a20cb6bee4673cd")], parse("""\
+ asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 --hash=sha256:eca58eb564b134e4ff521a02aa6f566c653835753e1fc8a50a20cb6bee4673cd")], parse_requirements_txt("""\
requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 \
--hash=sha256:eca58eb564b134e4ff521a02aa6f566c653835753e1fc8a50a20cb6bee4673cd
# via requirements.txt
""").requirements)
# Options
- asserts.equals(env, ["--pre"], parse("--pre\n").options)
- asserts.equals(env, ["--find-links", "/my/local/archives"], parse("--find-links /my/local/archives\n").options)
- asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse("""\
+ asserts.equals(env, ["--pre"], parse_requirements_txt("--pre\n").options)
+ asserts.equals(env, ["--find-links", "/my/local/archives"], parse_requirements_txt("--find-links /my/local/archives\n").options)
+ asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse_requirements_txt("""\
--pre
--find-links /my/local/archives
""").options)
- asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse("""\
+ asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse_requirements_txt("""\
--pre # Comment
--find-links /my/local/archives
""").options)
- asserts.equals(env, struct(requirements = [("FooProject", "FooProject==1.0.0")], options = ["--pre", "--find-links", "/my/local/archives"]), parse("""\
+ asserts.equals(env, struct(requirements = [("FooProject", "FooProject==1.0.0")], options = ["--pre", "--find-links", "/my/local/archives"]), parse_requirements_txt("""\
--pre # Comment
FooProject==1.0.0
--find-links /my/local/archives
@@ -116,7 +116,7 @@ def _parse_requirements_lockfile_test_impl(ctx):
("urllib3", "urllib3==1.26.7 --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"),
("yamllint", "yamllint==1.26.3 --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e"),
("setuptools", "setuptools==59.6.0 --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e"),
- ], parse("""\
+ ], parse_requirements_txt("""\
#
# This file is autogenerated by pip-compile with python 3.9
# To update, run:
@@ -220,5 +220,5 @@ parse_requirements_lockfile_test = unittest.make(
attrs = {},
)
-def parse_tests(name):
+def parse_requirements_txt_test_suite(name):
unittest.suite(name, parse_basic_test, parse_requirements_lockfile_test)
From b24f3c20705d1fd252a094e0c3d2ee60f6efc2c1 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:07:15 +0900
Subject: [PATCH 05/25] move parse_whl_name to private/pypi
---
python/pip_install/BUILD.bazel | 2 +-
python/pip_install/pip_repository.bzl | 2 +-
python/private/BUILD.bazel | 13 +++----------
python/private/bzlmod/BUILD.bazel | 2 +-
python/private/bzlmod/pip.bzl | 2 +-
python/private/patch_whl.bzl | 2 +-
python/private/pip_repo_name.bzl | 2 +-
python/private/pypi/BUILD.bazel | 5 +++++
python/private/{ => pypi}/parse_whl_name.bzl | 0
python/private/render_pkg_aliases.bzl | 2 +-
python/private/whl_target_platforms.bzl | 2 +-
tests/{private => pypi}/parse_whl_name/BUILD.bazel | 0
.../parse_whl_name/parse_whl_name_tests.bzl | 2 +-
13 files changed, 17 insertions(+), 19 deletions(-)
rename python/private/{ => pypi}/parse_whl_name.bzl (100%)
rename tests/{private => pypi}/parse_whl_name/BUILD.bazel (100%)
rename tests/{private => pypi}/parse_whl_name/parse_whl_name_tests.bzl (96%)
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 961027f706..cf863d8219 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -32,12 +32,12 @@ bzl_library(
"//python/private:envsubst_bzl",
"//python/private:normalize_name_bzl",
"//python/private:parse_requirements_bzl",
- "//python/private:parse_whl_name_bzl",
"//python/private:patch_whl_bzl",
"//python/private:render_pkg_aliases_bzl",
"//python/private:repo_utils_bzl",
"//python/private:toolchains_repo_bzl",
"//python/private:whl_target_platforms_bzl",
+ "//python/private/pypi:parse_whl_name_bzl",
"@bazel_skylib//lib:sets",
],
)
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index d6c8d9149c..6f4bf1d29f 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -25,12 +25,12 @@ load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
load("//python/private:envsubst.bzl", "envsubst")
load("//python/private:normalize_name.bzl", "normalize_name")
load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
-load("//python/private:parse_whl_name.bzl", "parse_whl_name")
load("//python/private:patch_whl.bzl", "patch_whl")
load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
load("//python/private:whl_target_platforms.bzl", "whl_target_platforms")
+load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
CPPFLAGS = "CPPFLAGS"
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 1524e42acd..1cdbc20ce3 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -129,7 +129,7 @@ bzl_library(
bzl_library(
name = "patch_whl_bzl",
srcs = ["patch_whl.bzl"],
- deps = [":parse_whl_name_bzl"],
+ deps = ["//python/private/pypi:parse_whl_name_bzl"],
)
bzl_library(
@@ -143,11 +143,6 @@ bzl_library(
],
)
-bzl_library(
- name = "parse_whl_name_bzl",
- srcs = ["parse_whl_name.bzl"],
-)
-
bzl_library(
name = "pip_flags_bzl",
srcs = ["pip_flags.bzl"],
@@ -161,7 +156,7 @@ bzl_library(
srcs = ["pip_repo_name.bzl"],
deps = [
":normalize_name_bzl",
- ":parse_whl_name_bzl",
+ "//python/private/pypi:parse_whl_name_bzl",
],
)
@@ -339,9 +334,7 @@ bzl_library(
name = "whl_target_platforms_bzl",
srcs = ["whl_target_platforms.bzl"],
visibility = ["//:__subpackages__"],
- deps = [
- "parse_whl_name_bzl",
- ],
+ deps = ["//python/private/pypi:parse_whl_name_bzl"],
)
bzl_library(
diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel
index 3362f34ffd..da024f0e61 100644
--- a/python/private/bzlmod/BUILD.bazel
+++ b/python/private/bzlmod/BUILD.bazel
@@ -35,7 +35,7 @@ bzl_library(
"//python/private:full_version_bzl",
"//python/private:normalize_name_bzl",
"//python/private:parse_requirements_bzl",
- "//python/private:parse_whl_name_bzl",
+ "//python/private/pypi:parse_whl_name_bzl",
"//python/private:pip_repo_name_bzl",
"//python/private:version_label_bzl",
":bazel_features_bzl",
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index 2ded3a4acd..d45270ed0e 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -25,12 +25,12 @@ load(
load("//python/private:auth.bzl", "AUTH_ATTRS")
load("//python/private:normalize_name.bzl", "normalize_name")
load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
-load("//python/private:parse_whl_name.bzl", "parse_whl_name")
load("//python/private:pip_repo_name.bzl", "pip_repo_name")
load("//python/private:pypi_index.bzl", "simpleapi_download")
load("//python/private:render_pkg_aliases.bzl", "whl_alias")
load("//python/private:repo_utils.bzl", "repo_utils")
load("//python/private:version_label.bzl", "version_label")
+load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
load(":pip_repository.bzl", "pip_repository")
def _parse_version(version):
diff --git a/python/private/patch_whl.bzl b/python/private/patch_whl.bzl
index 9e3119f744..0f5d2f7727 100644
--- a/python/private/patch_whl.bzl
+++ b/python/private/patch_whl.bzl
@@ -27,7 +27,7 @@ other patches ensures that the users have overview on exactly what has changed
within the wheel.
"""
-load("//python/private:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
_rules_python_root = Label("//:BUILD.bazel")
diff --git a/python/private/pip_repo_name.bzl b/python/private/pip_repo_name.bzl
index bef4304e15..28e4a6e611 100644
--- a/python/private/pip_repo_name.bzl
+++ b/python/private/pip_repo_name.bzl
@@ -15,8 +15,8 @@
"""A function to convert a dist name to a valid bazel repo name.
"""
+load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
load(":normalize_name.bzl", "normalize_name")
-load(":parse_whl_name.bzl", "parse_whl_name")
def pip_repo_name(prefix, filename, sha256):
"""Return a valid whl_library repo name given a distribution filename.
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 4d9a7164e9..a0210e4ec1 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -40,3 +40,8 @@ bzl_library(
name = "parse_requirements_txt_bzl",
srcs = ["parse_requirements_txt.bzl"],
)
+
+bzl_library(
+ name = "parse_whl_name_bzl",
+ srcs = ["parse_whl_name.bzl"],
+)
diff --git a/python/private/parse_whl_name.bzl b/python/private/pypi/parse_whl_name.bzl
similarity index 100%
rename from python/private/parse_whl_name.bzl
rename to python/private/pypi/parse_whl_name.bzl
diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl
index 82ac764f00..ceafc2b1a2 100644
--- a/python/private/render_pkg_aliases.bzl
+++ b/python/private/render_pkg_aliases.bzl
@@ -20,6 +20,7 @@ load(
"//python/pip_install/private:generate_group_library_build_bazel.bzl",
"generate_group_library_build_bazel",
) # buildifier: disable=bzl-visibility
+load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
load(
":labels.bzl",
"DATA_LABEL",
@@ -30,7 +31,6 @@ load(
"WHEEL_FILE_PUBLIC_LABEL",
)
load(":normalize_name.bzl", "normalize_name")
-load(":parse_whl_name.bzl", "parse_whl_name")
load(":text_util.bzl", "render")
load(":whl_target_platforms.bzl", "whl_target_platforms")
diff --git a/python/private/whl_target_platforms.bzl b/python/private/whl_target_platforms.bzl
index bee795732a..1970667d60 100644
--- a/python/private/whl_target_platforms.bzl
+++ b/python/private/whl_target_platforms.bzl
@@ -16,7 +16,7 @@
A starlark implementation of the wheel platform tag parsing to get the target platform.
"""
-load(":parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
# The order of the dictionaries is to keep definitions with their aliases next to each
# other
diff --git a/tests/private/parse_whl_name/BUILD.bazel b/tests/pypi/parse_whl_name/BUILD.bazel
similarity index 100%
rename from tests/private/parse_whl_name/BUILD.bazel
rename to tests/pypi/parse_whl_name/BUILD.bazel
diff --git a/tests/private/parse_whl_name/parse_whl_name_tests.bzl b/tests/pypi/parse_whl_name/parse_whl_name_tests.bzl
similarity index 96%
rename from tests/private/parse_whl_name/parse_whl_name_tests.bzl
rename to tests/pypi/parse_whl_name/parse_whl_name_tests.bzl
index c249f9fb1a..4a88a6e7c5 100644
--- a/tests/private/parse_whl_name/parse_whl_name_tests.bzl
+++ b/tests/pypi/parse_whl_name/parse_whl_name_tests.bzl
@@ -15,7 +15,7 @@
""
load("@rules_testing//lib:test_suite.bzl", "test_suite")
-load("//python/private:parse_whl_name.bzl", "parse_whl_name") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name") # buildifier: disable=bzl-visibility
_tests = []
From 11fd5ef4cf2352181e39c8f74c28ce4b9e4f77a1 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:13:18 +0900
Subject: [PATCH 06/25] move whl_target_platforms to private/pypi
---
python/pip_install/BUILD.bazel | 2 +-
python/pip_install/pip_repository.bzl | 2 +-
python/private/BUILD.bazel | 9 +--------
python/private/parse_requirements.bzl | 2 +-
python/private/pypi/BUILD.bazel | 6 ++++++
python/private/{ => pypi}/whl_target_platforms.bzl | 2 +-
python/private/render_pkg_aliases.bzl | 2 +-
tests/{private => pypi}/whl_target_platforms/BUILD.bazel | 0
.../whl_target_platforms/select_whl_tests.bzl | 2 +-
.../whl_target_platforms/whl_target_platforms_tests.bzl | 2 +-
10 files changed, 14 insertions(+), 15 deletions(-)
rename python/private/{ => pypi}/whl_target_platforms.bzl (99%)
rename tests/{private => pypi}/whl_target_platforms/BUILD.bazel (100%)
rename tests/{private => pypi}/whl_target_platforms/select_whl_tests.bzl (98%)
rename tests/{private => pypi}/whl_target_platforms/whl_target_platforms_tests.bzl (97%)
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index cf863d8219..d211300323 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -36,8 +36,8 @@ bzl_library(
"//python/private:render_pkg_aliases_bzl",
"//python/private:repo_utils_bzl",
"//python/private:toolchains_repo_bzl",
- "//python/private:whl_target_platforms_bzl",
"//python/private/pypi:parse_whl_name_bzl",
+ "//python/private/pypi:whl_target_platforms_bzl",
"@bazel_skylib//lib:sets",
],
)
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index 6f4bf1d29f..6afd70e97c 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -29,8 +29,8 @@ load("//python/private:patch_whl.bzl", "patch_whl")
load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
-load("//python/private:whl_target_platforms.bzl", "whl_target_platforms")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
CPPFLAGS = "CPPFLAGS"
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 1cdbc20ce3..4af813c71d 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -137,9 +137,9 @@ bzl_library(
srcs = ["parse_requirements.bzl"],
deps = [
":normalize_name_bzl",
- ":whl_target_platforms_bzl",
"//python/private/pypi:index_sources_bzl",
"//python/private/pypi:parse_requirements_txt_bzl",
+ "//python/private/pypi:whl_target_platforms_bzl",
],
)
@@ -330,13 +330,6 @@ bzl_library(
srcs = ["version_label.bzl"],
)
-bzl_library(
- name = "whl_target_platforms_bzl",
- srcs = ["whl_target_platforms.bzl"],
- visibility = ["//:__subpackages__"],
- deps = ["//python/private/pypi:parse_whl_name_bzl"],
-)
-
bzl_library(
name = "labels_bzl",
srcs = ["labels.bzl"],
diff --git a/python/private/parse_requirements.bzl b/python/private/parse_requirements.bzl
index 1b14ae0dc1..07fd0f5143 100644
--- a/python/private/parse_requirements.bzl
+++ b/python/private/parse_requirements.bzl
@@ -28,8 +28,8 @@ behavior.
load("//python/private/pypi:index_sources.bzl", "index_sources")
load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt")
+load("//python/private/pypi:whl_target_platforms.bzl", "select_whls", "whl_target_platforms")
load(":normalize_name.bzl", "normalize_name")
-load(":whl_target_platforms.bzl", "select_whls", "whl_target_platforms")
# This includes the vendored _translate_cpu and _translate_os from
# @platforms//host:extension.bzl at version 0.0.9 so that we don't
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index a0210e4ec1..0cf08a74ad 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -45,3 +45,9 @@ bzl_library(
name = "parse_whl_name_bzl",
srcs = ["parse_whl_name.bzl"],
)
+
+bzl_library(
+ name = "whl_target_platforms_bzl",
+ srcs = ["whl_target_platforms.bzl"],
+ deps = [":parse_whl_name_bzl"],
+)
diff --git a/python/private/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl
similarity index 99%
rename from python/private/whl_target_platforms.bzl
rename to python/private/pypi/whl_target_platforms.bzl
index 1970667d60..bee795732a 100644
--- a/python/private/whl_target_platforms.bzl
+++ b/python/private/pypi/whl_target_platforms.bzl
@@ -16,7 +16,7 @@
A starlark implementation of the wheel platform tag parsing to get the target platform.
"""
-load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load(":parse_whl_name.bzl", "parse_whl_name")
# The order of the dictionaries is to keep definitions with their aliases next to each
# other
diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl
index ceafc2b1a2..76d33243dd 100644
--- a/python/private/render_pkg_aliases.bzl
+++ b/python/private/render_pkg_aliases.bzl
@@ -21,6 +21,7 @@ load(
"generate_group_library_build_bazel",
) # buildifier: disable=bzl-visibility
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
load(
":labels.bzl",
"DATA_LABEL",
@@ -32,7 +33,6 @@ load(
)
load(":normalize_name.bzl", "normalize_name")
load(":text_util.bzl", "render")
-load(":whl_target_platforms.bzl", "whl_target_platforms")
NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\
No matching wheel for current configuration's Python version.
diff --git a/tests/private/whl_target_platforms/BUILD.bazel b/tests/pypi/whl_target_platforms/BUILD.bazel
similarity index 100%
rename from tests/private/whl_target_platforms/BUILD.bazel
rename to tests/pypi/whl_target_platforms/BUILD.bazel
diff --git a/tests/private/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/whl_target_platforms/select_whl_tests.bzl
similarity index 98%
rename from tests/private/whl_target_platforms/select_whl_tests.bzl
rename to tests/pypi/whl_target_platforms/select_whl_tests.bzl
index 59e9d77918..3fd80e3ee3 100644
--- a/tests/private/whl_target_platforms/select_whl_tests.bzl
+++ b/tests/pypi/whl_target_platforms/select_whl_tests.bzl
@@ -15,7 +15,7 @@
""
load("@rules_testing//lib:test_suite.bzl", "test_suite")
-load("//python/private:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility
WHL_LIST = [
"pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl",
diff --git a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl b/tests/pypi/whl_target_platforms/whl_target_platforms_tests.bzl
similarity index 97%
rename from tests/private/whl_target_platforms/whl_target_platforms_tests.bzl
rename to tests/pypi/whl_target_platforms/whl_target_platforms_tests.bzl
index 07f3158b31..a72bdc275f 100644
--- a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl
+++ b/tests/pypi/whl_target_platforms/whl_target_platforms_tests.bzl
@@ -15,7 +15,7 @@
""
load("@rules_testing//lib:test_suite.bzl", "test_suite")
-load("//python/private:whl_target_platforms.bzl", "whl_target_platforms") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms") # buildifier: disable=bzl-visibility
_tests = []
From 0e9254d01253835f91310a953abbe9c2648c78d3 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:20:17 +0900
Subject: [PATCH 07/25] move parse_requirements to private/pypi
---
.github/CODEOWNERS | 1 +
python/pip_install/BUILD.bazel | 2 +-
python/pip_install/pip_repository.bzl | 2 +-
python/private/BUILD.bazel | 11 -----------
python/private/bzlmod/BUILD.bazel | 2 +-
python/private/bzlmod/pip.bzl | 2 +-
python/private/pypi/BUILD.bazel | 11 +++++++++++
python/private/{ => pypi}/parse_requirements.bzl | 8 ++++----
.../{private => pypi}/parse_requirements/BUILD.bazel | 0
.../parse_requirements/parse_requirements_tests.bzl | 2 +-
10 files changed, 21 insertions(+), 20 deletions(-)
rename python/private/{ => pypi}/parse_requirements.bzl (98%)
rename tests/{private => pypi}/parse_requirements/BUILD.bazel (100%)
rename tests/{private => pypi}/parse_requirements/parse_requirements_tests.bzl (99%)
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 4f0ca4d0a8..a75b5a91a3 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -13,3 +13,4 @@
# PyPI integration related code
/python/private/pypi/ @aignas
+/tests/pypi/ @aignas
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index d211300323..79b6598367 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -31,11 +31,11 @@ bzl_library(
"//python/private:bzlmod_enabled_bzl",
"//python/private:envsubst_bzl",
"//python/private:normalize_name_bzl",
- "//python/private:parse_requirements_bzl",
"//python/private:patch_whl_bzl",
"//python/private:render_pkg_aliases_bzl",
"//python/private:repo_utils_bzl",
"//python/private:toolchains_repo_bzl",
+ "//python/private/pypi:parse_requirements_bzl",
"//python/private/pypi:parse_whl_name_bzl",
"//python/private/pypi:whl_target_platforms_bzl",
"@bazel_skylib//lib:sets",
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index 6afd70e97c..d8a5f56193 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -24,11 +24,11 @@ load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
load("//python/private:envsubst.bzl", "envsubst")
load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private:patch_whl.bzl", "patch_whl")
load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
+load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 4af813c71d..1cbb1021b3 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -132,17 +132,6 @@ bzl_library(
deps = ["//python/private/pypi:parse_whl_name_bzl"],
)
-bzl_library(
- name = "parse_requirements_bzl",
- srcs = ["parse_requirements.bzl"],
- deps = [
- ":normalize_name_bzl",
- "//python/private/pypi:index_sources_bzl",
- "//python/private/pypi:parse_requirements_txt_bzl",
- "//python/private/pypi:whl_target_platforms_bzl",
- ],
-)
-
bzl_library(
name = "pip_flags_bzl",
srcs = ["pip_flags.bzl"],
diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel
index da024f0e61..cb3d4b3639 100644
--- a/python/private/bzlmod/BUILD.bazel
+++ b/python/private/bzlmod/BUILD.bazel
@@ -34,7 +34,7 @@ bzl_library(
"//python/private:pypi_index_bzl",
"//python/private:full_version_bzl",
"//python/private:normalize_name_bzl",
- "//python/private:parse_requirements_bzl",
+ "//python/private/pypi:parse_requirements_bzl",
"//python/private/pypi:parse_whl_name_bzl",
"//python/private:pip_repo_name_bzl",
"//python/private:version_label_bzl",
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index d45270ed0e..8e5ef92133 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -24,12 +24,12 @@ load(
)
load("//python/private:auth.bzl", "AUTH_ATTRS")
load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private:pip_repo_name.bzl", "pip_repo_name")
load("//python/private:pypi_index.bzl", "simpleapi_download")
load("//python/private:render_pkg_aliases.bzl", "whl_alias")
load("//python/private:repo_utils.bzl", "repo_utils")
load("//python/private:version_label.bzl", "version_label")
+load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
load(":pip_repository.bzl", "pip_repository")
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 0cf08a74ad..b9b0e4fe4d 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -36,6 +36,17 @@ bzl_library(
srcs = ["index_sources.bzl"],
)
+bzl_library(
+ name = "parse_requirements_bzl",
+ srcs = ["parse_requirements.bzl"],
+ deps = [
+ ":index_sources_bzl",
+ ":parse_requirements_txt_bzl",
+ ":whl_target_platforms_bzl",
+ "//python/private:normalize_name_bzl",
+ ],
+)
+
bzl_library(
name = "parse_requirements_txt_bzl",
srcs = ["parse_requirements_txt.bzl"],
diff --git a/python/private/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
similarity index 98%
rename from python/private/parse_requirements.bzl
rename to python/private/pypi/parse_requirements.bzl
index 07fd0f5143..22a6f0a875 100644
--- a/python/private/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -26,10 +26,10 @@ file for the host platform to be backwards compatible with the legacy
behavior.
"""
-load("//python/private/pypi:index_sources.bzl", "index_sources")
-load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt")
-load("//python/private/pypi:whl_target_platforms.bzl", "select_whls", "whl_target_platforms")
-load(":normalize_name.bzl", "normalize_name")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load(":index_sources.bzl", "index_sources")
+load(":parse_requirements_txt.bzl", "parse_requirements_txt")
+load(":whl_target_platforms.bzl", "select_whls", "whl_target_platforms")
# This includes the vendored _translate_cpu and _translate_os from
# @platforms//host:extension.bzl at version 0.0.9 so that we don't
diff --git a/tests/private/parse_requirements/BUILD.bazel b/tests/pypi/parse_requirements/BUILD.bazel
similarity index 100%
rename from tests/private/parse_requirements/BUILD.bazel
rename to tests/pypi/parse_requirements/BUILD.bazel
diff --git a/tests/private/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
similarity index 99%
rename from tests/private/parse_requirements/parse_requirements_tests.bzl
rename to tests/pypi/parse_requirements/parse_requirements_tests.bzl
index 81cf523460..13ce8f44ed 100644
--- a/tests/private/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -15,7 +15,7 @@
""
load("@rules_testing//lib:test_suite.bzl", "test_suite")
-load("//python/private:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility
def _mock_ctx():
testdata = {
From 4504416897fe45e0cad47d575c548787c288b033 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:27:26 +0900
Subject: [PATCH 08/25] move pip_repo_name to private/pypi
---
python/private/BUILD.bazel | 9 ---------
python/private/bzlmod/BUILD.bazel | 2 +-
python/private/bzlmod/pip.bzl | 4 ++--
python/private/pypi/BUILD.bazel | 9 +++++++++
.../{pip_repo_name.bzl => pypi/whl_repo_name.bzl} | 6 +++---
tests/private/pip_repo_name/BUILD.bazel | 3 ---
tests/pypi/whl_repo_name/BUILD.bazel | 3 +++
.../whl_repo_name/whl_repo_name_tests.bzl} | 10 +++++-----
8 files changed, 23 insertions(+), 23 deletions(-)
rename python/private/{pip_repo_name.bzl => pypi/whl_repo_name.bzl} (91%)
delete mode 100644 tests/private/pip_repo_name/BUILD.bazel
create mode 100644 tests/pypi/whl_repo_name/BUILD.bazel
rename tests/{private/pip_repo_name/pip_repo_name_tests.bzl => pypi/whl_repo_name/whl_repo_name_tests.bzl} (82%)
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 1cbb1021b3..0e19623b38 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -140,15 +140,6 @@ bzl_library(
],
)
-bzl_library(
- name = "pip_repo_name_bzl",
- srcs = ["pip_repo_name.bzl"],
- deps = [
- ":normalize_name_bzl",
- "//python/private/pypi:parse_whl_name_bzl",
- ],
-)
-
bzl_library(
name = "pypi_index_bzl",
srcs = ["pypi_index.bzl"],
diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel
index cb3d4b3639..d1e6e5964f 100644
--- a/python/private/bzlmod/BUILD.bazel
+++ b/python/private/bzlmod/BUILD.bazel
@@ -36,7 +36,7 @@ bzl_library(
"//python/private:normalize_name_bzl",
"//python/private/pypi:parse_requirements_bzl",
"//python/private/pypi:parse_whl_name_bzl",
- "//python/private:pip_repo_name_bzl",
+ "//python/private/pypi:whl_repo_name_bzl",
"//python/private:version_label_bzl",
":bazel_features_bzl",
] + [
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index 8e5ef92133..2b8ecd5403 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -24,13 +24,13 @@ load(
)
load("//python/private:auth.bzl", "AUTH_ATTRS")
load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:pip_repo_name.bzl", "pip_repo_name")
load("//python/private:pypi_index.bzl", "simpleapi_download")
load("//python/private:render_pkg_aliases.bzl", "whl_alias")
load("//python/private:repo_utils.bzl", "repo_utils")
load("//python/private:version_label.bzl", "version_label")
load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:whl_repo_name.bzl", "whl_repo_name")
load(":pip_repository.bzl", "pip_repository")
def _parse_version(version):
@@ -268,7 +268,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
# This is no-op because pip is not used to download the wheel.
whl_library_args.pop("download_only", None)
- repo_name = pip_repo_name(pip_name, distribution.filename, distribution.sha256)
+ repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256)
whl_library_args["requirement"] = requirement.srcs.requirement
whl_library_args["urls"] = [distribution.url]
whl_library_args["sha256"] = distribution.sha256
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index b9b0e4fe4d..936f7fc292 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -57,6 +57,15 @@ bzl_library(
srcs = ["parse_whl_name.bzl"],
)
+bzl_library(
+ name = "whl_repo_name",
+ srcs = ["whl_repo_name.bzl"],
+ deps = [
+ ":parse_whl_name_bzl",
+ "//python/private:normalize_name_bzl",
+ ],
+)
+
bzl_library(
name = "whl_target_platforms_bzl",
srcs = ["whl_target_platforms.bzl"],
diff --git a/python/private/pip_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl
similarity index 91%
rename from python/private/pip_repo_name.bzl
rename to python/private/pypi/whl_repo_name.bzl
index 28e4a6e611..295f5a45c4 100644
--- a/python/private/pip_repo_name.bzl
+++ b/python/private/pypi/whl_repo_name.bzl
@@ -15,10 +15,10 @@
"""A function to convert a dist name to a valid bazel repo name.
"""
-load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
-load(":normalize_name.bzl", "normalize_name")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load(":parse_whl_name.bzl", "parse_whl_name")
-def pip_repo_name(prefix, filename, sha256):
+def whl_repo_name(prefix, filename, sha256):
"""Return a valid whl_library repo name given a distribution filename.
Args:
diff --git a/tests/private/pip_repo_name/BUILD.bazel b/tests/private/pip_repo_name/BUILD.bazel
deleted file mode 100644
index 7c6782daaf..0000000000
--- a/tests/private/pip_repo_name/BUILD.bazel
+++ /dev/null
@@ -1,3 +0,0 @@
-load(":pip_repo_name_tests.bzl", "pip_repo_name_test_suite")
-
-pip_repo_name_test_suite(name = "pip_repo_name_tests")
diff --git a/tests/pypi/whl_repo_name/BUILD.bazel b/tests/pypi/whl_repo_name/BUILD.bazel
new file mode 100644
index 0000000000..8671dd7754
--- /dev/null
+++ b/tests/pypi/whl_repo_name/BUILD.bazel
@@ -0,0 +1,3 @@
+load(":whl_repo_name_tests.bzl", "whl_repo_name_test_suite")
+
+whl_repo_name_test_suite(name = "whl_repo_name_tests")
diff --git a/tests/private/pip_repo_name/pip_repo_name_tests.bzl b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl
similarity index 82%
rename from tests/private/pip_repo_name/pip_repo_name_tests.bzl
rename to tests/pypi/whl_repo_name/whl_repo_name_tests.bzl
index 574d558277..8b7df83530 100644
--- a/tests/private/pip_repo_name/pip_repo_name_tests.bzl
+++ b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl
@@ -15,24 +15,24 @@
""
load("@rules_testing//lib:test_suite.bzl", "test_suite")
-load("//python/private:pip_repo_name.bzl", "pip_repo_name") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:whl_repo_name.bzl", "whl_repo_name") # buildifier: disable=bzl-visibility
_tests = []
def _test_simple(env):
- got = pip_repo_name("prefix", "foo-1.2.3-py3-none-any.whl", "deadbeef")
+ got = whl_repo_name("prefix", "foo-1.2.3-py3-none-any.whl", "deadbeef")
env.expect.that_str(got).equals("prefix_foo_py3_none_any_deadbeef")
_tests.append(_test_simple)
def _test_sdist(env):
- got = pip_repo_name("prefix", "foo-1.2.3.tar.gz", "deadbeef000deadbeef")
+ got = whl_repo_name("prefix", "foo-1.2.3.tar.gz", "deadbeef000deadbeef")
env.expect.that_str(got).equals("prefix_foo_sdist_deadbeef")
_tests.append(_test_sdist)
def _test_platform_whl(env):
- got = pip_repo_name(
+ got = whl_repo_name(
"prefix",
"foo-1.2.3-cp39.cp310-abi3-manylinux1_x86_64.manylinux_2_17_x86_64.whl",
"deadbeef000deadbeef",
@@ -43,7 +43,7 @@ def _test_platform_whl(env):
_tests.append(_test_platform_whl)
-def pip_repo_name_test_suite(name):
+def whl_repo_name_test_suite(name):
"""Create the test suite.
Args:
From 37be3ccab8bb200424c4c0bf073f1959f7d7547c Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:29:15 +0900
Subject: [PATCH 09/25] remove unused file
---
python/private/normalize_platform.bzl | 13 -------------
1 file changed, 13 deletions(-)
delete mode 100644 python/private/normalize_platform.bzl
diff --git a/python/private/normalize_platform.bzl b/python/private/normalize_platform.bzl
deleted file mode 100644
index 633062f399..0000000000
--- a/python/private/normalize_platform.bzl
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
From 82c81d21daf72afa5048a18190621a0e68be5f2a Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:42:24 +0900
Subject: [PATCH 10/25] move pip_config_settings to private/pypi
---
python/config_settings/BUILD.bazel | 2 +-
python/private/BUILD.bazel | 8 --------
python/private/pypi/BUILD.bazel | 12 ++++++++++++
.../config_settings.bzl} | 9 ++-------
python/private/{pip_flags.bzl => pypi/flags.bzl} | 2 +-
python/private/render_pkg_aliases.bzl | 10 +++++-----
.../render_pkg_aliases/render_pkg_aliases_test.bzl | 14 +++++---------
tests/private/pip_config_settings/BUILD.bazel | 5 -----
tests/pypi/config_settings/BUILD.bazel | 5 +++++
.../config_settings/config_settings_tests.bzl} | 6 +++---
10 files changed, 34 insertions(+), 39 deletions(-)
rename python/private/{pip_config_settings.bzl => pypi/config_settings.bzl} (98%)
rename python/private/{pip_flags.bzl => pypi/flags.bzl} (98%)
delete mode 100644 tests/private/pip_config_settings/BUILD.bazel
create mode 100644 tests/pypi/config_settings/BUILD.bazel
rename tests/{private/pip_config_settings/pip_config_settings_tests.bzl => pypi/config_settings/config_settings_tests.bzl} (98%)
diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel
index 1003efb8c2..f6f46db770 100644
--- a/python/config_settings/BUILD.bazel
+++ b/python/config_settings/BUILD.bazel
@@ -9,7 +9,7 @@ load(
"PycCollectionFlag",
)
load(
- "//python/private:pip_flags.bzl",
+ "//python/private/pypi:flags.bzl",
"INTERNAL_FLAGS",
"UniversalWhlFlag",
"UseWhlFlag",
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 0e19623b38..f0e29a2839 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -132,14 +132,6 @@ bzl_library(
deps = ["//python/private/pypi:parse_whl_name_bzl"],
)
-bzl_library(
- name = "pip_flags_bzl",
- srcs = ["pip_flags.bzl"],
- deps = [
- ":enum_bzl",
- ],
-)
-
bzl_library(
name = "pypi_index_bzl",
srcs = ["pypi_index.bzl"],
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 936f7fc292..cf66161021 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -31,6 +31,18 @@ filegroup(
visibility = ["//python/private:__pkg__"],
)
+bzl_library(
+ name = "config_settings_bzl",
+ srcs = ["config_settings.bzl"],
+ deps = ["flags_bzl"],
+)
+
+bzl_library(
+ name = "flags_bzl",
+ srcs = ["flags.bzl"],
+ deps = ["//python/private:enum_bzl"],
+)
+
bzl_library(
name = "index_sources_bzl",
srcs = ["index_sources.bzl"],
diff --git a/python/private/pip_config_settings.bzl b/python/private/pypi/config_settings.bzl
similarity index 98%
rename from python/private/pip_config_settings.bzl
rename to python/private/pypi/config_settings.bzl
index b13b39b725..974121782f 100644
--- a/python/private/pip_config_settings.bzl
+++ b/python/private/pypi/config_settings.bzl
@@ -39,12 +39,7 @@ Note, that here the specialization of musl vs manylinux wheels is the same in
order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa.
"""
-load(
- ":pip_flags.bzl",
- "INTERNAL_FLAGS",
- "UniversalWhlFlag",
- "WhlLibcFlag",
-)
+load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag", "WhlLibcFlag")
FLAGS = struct(
**{
@@ -73,7 +68,7 @@ _flags = struct(
}
)
-def pip_config_settings(
+def config_settings(
*,
python_versions = [],
glibc_versions = [],
diff --git a/python/private/pip_flags.bzl b/python/private/pypi/flags.bzl
similarity index 98%
rename from python/private/pip_flags.bzl
rename to python/private/pypi/flags.bzl
index 1d271c7318..d834be8cc6 100644
--- a/python/private/pip_flags.bzl
+++ b/python/private/pypi/flags.bzl
@@ -18,7 +18,7 @@ NOTE: The transitive loads of this should be kept minimal. This avoids loading
unnecessary files when all that are needed are flag definitions.
"""
-load(":enum.bzl", "enum")
+load("//python/private:enum.bzl", "enum")
# Determines if we should use whls for third party
#
diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl
index 76d33243dd..5c67d8c87f 100644
--- a/python/private/render_pkg_aliases.bzl
+++ b/python/private/render_pkg_aliases.bzl
@@ -309,7 +309,7 @@ def render_multiplatform_pkg_aliases(*, aliases, default_version = None, **kwarg
aliases = config_setting_aliases,
**kwargs
)
- contents["_config/BUILD.bazel"] = _render_pip_config_settings(**flag_versions)
+ contents["_config/BUILD.bazel"] = _render_config_settings(**flag_versions)
return contents
def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs):
@@ -398,12 +398,12 @@ def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs):
ret.extend(versioned.values())
return ret
-def _render_pip_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []):
+def _render_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []):
return """\
-load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings")
+load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings")
-pip_config_settings(
- name = "pip_config_settings",
+config_settings(
+ name = "config_settings",
glibc_versions = {glibc_versions},
muslc_versions = {muslc_versions},
osx_versions = {osx_versions},
diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl
index da8a918aed..3e0ac6d609 100644
--- a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl
+++ b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl
@@ -16,10 +16,6 @@
load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility
-load(
- "//python/private:pip_config_settings.bzl",
- "pip_config_settings",
-) # buildifier: disable=bzl-visibility
load(
"//python/private:render_pkg_aliases.bzl",
"get_filename_config_settings",
@@ -29,6 +25,7 @@ load(
"render_pkg_aliases",
"whl_alias",
) # buildifier: disable=bzl-visibility
+load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility
def _normalize_label_strings(want):
"""normalize expected strings.
@@ -182,10 +179,10 @@ alias(
env.expect.that_str(actual.pop("_config/BUILD.bazel")).equals(
"""\
-load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings")
+load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings")
-pip_config_settings(
- name = "pip_config_settings",
+config_settings(
+ name = "config_settings",
glibc_versions = [],
muslc_versions = [],
osx_versions = [],
@@ -722,7 +719,6 @@ def _test_cp37_abi3_linux_x86_64(env):
python_default = True,
want = [
":is_cp3x_abi3_linux_x86_64",
- # TODO @aignas 2024-05-29: update the pip_config_settings to generate this
":is_cp3.2_cp3x_abi3_linux_x86_64",
],
)
@@ -924,7 +920,7 @@ def _test_config_settings_exist(env):
]
available_config_settings = []
mock_rule = lambda name, **kwargs: available_config_settings.append(name)
- pip_config_settings(
+ config_settings(
python_versions = ["3.11"],
native = struct(
alias = mock_rule,
diff --git a/tests/private/pip_config_settings/BUILD.bazel b/tests/private/pip_config_settings/BUILD.bazel
deleted file mode 100644
index c3752e0ac3..0000000000
--- a/tests/private/pip_config_settings/BUILD.bazel
+++ /dev/null
@@ -1,5 +0,0 @@
-load(":pip_config_settings_tests.bzl", "pip_config_settings_test_suite")
-
-pip_config_settings_test_suite(
- name = "pip_config_settings",
-)
diff --git a/tests/pypi/config_settings/BUILD.bazel b/tests/pypi/config_settings/BUILD.bazel
new file mode 100644
index 0000000000..15dbd7f70e
--- /dev/null
+++ b/tests/pypi/config_settings/BUILD.bazel
@@ -0,0 +1,5 @@
+load(":config_settings_tests.bzl", "config_settings_test_suite")
+
+config_settings_test_suite(
+ name = "config_settings_tests",
+)
diff --git a/tests/private/pip_config_settings/pip_config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl
similarity index 98%
rename from tests/private/pip_config_settings/pip_config_settings_tests.bzl
rename to tests/pypi/config_settings/config_settings_tests.bzl
index a66e7f47d5..87e18b412f 100644
--- a/tests/private/pip_config_settings/pip_config_settings_tests.bzl
+++ b/tests/pypi/config_settings/config_settings_tests.bzl
@@ -17,7 +17,7 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test")
load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("@rules_testing//lib:truth.bzl", "subjects")
load("@rules_testing//lib:util.bzl", test_util = "util")
-load("//python/private:pip_config_settings.bzl", "pip_config_settings") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility
def _subject_impl(ctx):
_ = ctx # @unused
@@ -520,13 +520,13 @@ def _test_all(name):
_tests.append(_test_all)
-def pip_config_settings_test_suite(name): # buildifier: disable=function-docstring
+def config_settings_test_suite(name): # buildifier: disable=function-docstring
test_suite(
name = name,
tests = _tests,
)
- pip_config_settings(
+ config_settings(
name = "dummy",
python_versions = ["3.8", "3.9", "3.10"],
glibc_versions = [(2, 14), (2, 17)],
From 70dd0a13af27bdbf02107754189673a1c1ce560b Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:53:30 +0900
Subject: [PATCH 11/25] move pypi_index to pypi/private and rename
---
python/private/BUILD.bazel | 11 --
python/private/bzlmod/BUILD.bazel | 2 +-
python/private/bzlmod/pip.bzl | 2 +-
python/private/pypi/BUILD.bazel | 17 +++
python/private/pypi/parse_simpleapi_html.bzl | 106 ++++++++++++++++++
.../simpleapi_download.bzl} | 102 ++---------------
tests/private/pypi_index/BUILD.bazel | 3 -
tests/pypi/parse_simpleapi_html/BUILD.bazel | 3 +
.../parse_simpleapi_html_tests.bzl} | 16 +--
9 files changed, 143 insertions(+), 119 deletions(-)
create mode 100644 python/private/pypi/parse_simpleapi_html.bzl
rename python/private/{pypi_index.bzl => pypi/simpleapi_download.bzl} (67%)
delete mode 100644 tests/private/pypi_index/BUILD.bazel
create mode 100644 tests/pypi/parse_simpleapi_html/BUILD.bazel
rename tests/{private/pypi_index/pypi_index_tests.bzl => pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl} (95%)
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index f0e29a2839..ce2c4172d3 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -132,17 +132,6 @@ bzl_library(
deps = ["//python/private/pypi:parse_whl_name_bzl"],
)
-bzl_library(
- name = "pypi_index_bzl",
- srcs = ["pypi_index.bzl"],
- deps = [
- ":auth_bzl",
- ":normalize_name_bzl",
- ":text_util_bzl",
- "//python/private/bzlmod:bazel_features_bzl",
- ],
-)
-
bzl_library(
name = "py_cc_toolchain_bzl",
srcs = [
diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel
index d1e6e5964f..dfc30173dd 100644
--- a/python/private/bzlmod/BUILD.bazel
+++ b/python/private/bzlmod/BUILD.bazel
@@ -31,7 +31,7 @@ bzl_library(
deps = [
":pip_repository_bzl",
"//python/pip_install:pip_repository_bzl",
- "//python/private:pypi_index_bzl",
+ "//python/private/pypi:simpleapi_download_bzl",
"//python/private:full_version_bzl",
"//python/private:normalize_name_bzl",
"//python/private/pypi:parse_requirements_bzl",
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index 2b8ecd5403..a57f42009d 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -24,12 +24,12 @@ load(
)
load("//python/private:auth.bzl", "AUTH_ATTRS")
load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:pypi_index.bzl", "simpleapi_download")
load("//python/private:render_pkg_aliases.bzl", "whl_alias")
load("//python/private:repo_utils.bzl", "repo_utils")
load("//python/private:version_label.bzl", "version_label")
load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:simpleapi_download.bzl", "simpleapi_download")
load("//python/private/pypi:whl_repo_name.bzl", "whl_repo_name")
load(":pip_repository.bzl", "pip_repository")
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index cf66161021..d39af23f6b 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -64,11 +64,28 @@ bzl_library(
srcs = ["parse_requirements_txt.bzl"],
)
+bzl_library(
+ name = "parse_simpleapi_html_bzl",
+ srcs = ["parse_simpleapi_html.bzl"],
+)
+
bzl_library(
name = "parse_whl_name_bzl",
srcs = ["parse_whl_name.bzl"],
)
+bzl_library(
+ name = "simpleapi_download_bzl",
+ srcs = ["simpleapi_download.bzl"],
+ deps = [
+ ":parse_simpleapi_html_bzl",
+ "//python/private:auth_bzl",
+ "//python/private:normalize_name_bzl",
+ "//python/private:text_util_bzl",
+ "//python/private/bzlmod:bazel_features_bzl",
+ ],
+)
+
bzl_library(
name = "whl_repo_name",
srcs = ["whl_repo_name.bzl"],
diff --git a/python/private/pypi/parse_simpleapi_html.bzl b/python/private/pypi/parse_simpleapi_html.bzl
new file mode 100644
index 0000000000..f7cd032aca
--- /dev/null
+++ b/python/private/pypi/parse_simpleapi_html.bzl
@@ -0,0 +1,106 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Parse SimpleAPI HTML in Starlark.
+"""
+
+def parse_simpleapi_html(*, url, content):
+ """Get the package URLs for given shas by parsing the Simple API HTML.
+
+ Args:
+ url(str): The URL that the HTML content can be downloaded from.
+ content(str): The Simple API HTML content.
+
+ Returns:
+ A list of structs with:
+ * filename: The filename of the artifact.
+ * url: The URL to download the artifact.
+ * sha256: The sha256 of the artifact.
+ * metadata_sha256: The whl METADATA sha256 if we can download it. If this is
+ present, then the 'metadata_url' is also present. Defaults to "".
+ * metadata_url: The URL for the METADATA if we can download it. Defaults to "".
+ """
+ sdists = {}
+ whls = {}
+ lines = content.split("= (2, 0):
+ # We don't expect to have version 2.0 here, but have this check in place just in case.
+ # https://packaging.python.org/en/latest/specifications/simple-repository-api/#versioning-pypi-s-simple-api
+ fail("Unsupported API version: {}".format(api_version))
+
+ for line in lines[1:]:
+ dist_url, _, tail = line.partition("#sha256=")
+ sha256, _, tail = tail.partition("\"")
+
+ # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
+ yanked = "data-yanked" in line
+
+ maybe_metadata, _, tail = tail.partition(">")
+ filename, _, tail = tail.partition("<")
+
+ metadata_sha256 = ""
+ metadata_url = ""
+ for metadata_marker in ["data-core-metadata", "data-dist-info-metadata"]:
+ metadata_marker = metadata_marker + "=\"sha256="
+ if metadata_marker in maybe_metadata:
+ # Implement https://peps.python.org/pep-0714/
+ _, _, tail = maybe_metadata.partition(metadata_marker)
+ metadata_sha256, _, _ = tail.partition("\"")
+ metadata_url = dist_url + ".metadata"
+ break
+
+ if filename.endswith(".whl"):
+ whls[sha256] = struct(
+ filename = filename,
+ url = _absolute_url(url, dist_url),
+ sha256 = sha256,
+ metadata_sha256 = metadata_sha256,
+ metadata_url = _absolute_url(url, metadata_url),
+ yanked = yanked,
+ )
+ else:
+ sdists[sha256] = struct(
+ filename = filename,
+ url = _absolute_url(url, dist_url),
+ sha256 = sha256,
+ metadata_sha256 = "",
+ metadata_url = "",
+ yanked = yanked,
+ )
+
+ return struct(
+ sdists = sdists,
+ whls = whls,
+ )
+
+def _absolute_url(index_url, candidate):
+ if not candidate.startswith(".."):
+ return candidate
+
+ candidate_parts = candidate.split("..")
+ last = candidate_parts[-1]
+ for _ in range(len(candidate_parts) - 1):
+ index_url, _, _ = index_url.rstrip("/").rpartition("/")
+
+ return "{}/{}".format(index_url, last.strip("/"))
diff --git a/python/private/pypi_index.bzl b/python/private/pypi/simpleapi_download.bzl
similarity index 67%
rename from python/private/pypi_index.bzl
rename to python/private/pypi/simpleapi_download.bzl
index 64d908e32b..b258fef07a 100644
--- a/python/private/pypi_index.bzl
+++ b/python/private/pypi/simpleapi_download.bzl
@@ -17,9 +17,10 @@ A file that houses private functions used in the `bzlmod` extension with the sam
"""
load("@bazel_features//:features.bzl", "bazel_features")
-load(":auth.bzl", "get_auth")
-load(":envsubst.bzl", "envsubst")
-load(":normalize_name.bzl", "normalize_name")
+load("//python/private:auth.bzl", "get_auth")
+load("//python/private:envsubst.bzl", "envsubst")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load(":parse_simpleapi_html.bzl", "parse_simpleapi_html")
def simpleapi_download(ctx, *, attr, cache, parallel_download = True):
"""Download Simple API HTML.
@@ -71,7 +72,7 @@ def simpleapi_download(ctx, *, attr, cache, parallel_download = True):
success = False
for index_url in index_urls:
- result = read_simple_api(
+ result = _read_simpleapi(
ctx = ctx,
url = "{}/{}/".format(
index_url_overrides.get(pkg_normalized, index_url).rstrip("/"),
@@ -122,7 +123,7 @@ def simpleapi_download(ctx, *, attr, cache, parallel_download = True):
return contents
-def read_simple_api(ctx, url, attr, cache, **download_kwargs):
+def _read_simpleapi(ctx, url, attr, cache, **download_kwargs):
"""Read SimpleAPI.
Args:
@@ -195,98 +196,9 @@ def _read_index_result(ctx, result, output, url, cache, cache_key):
content = ctx.read(output)
- output = parse_simple_api_html(url = url, content = content)
+ output = parse_simpleapi_html(url = url, content = content)
if output:
cache.setdefault(cache_key, output)
return struct(success = True, output = output, cache_key = cache_key)
else:
return struct(success = False)
-
-def parse_simple_api_html(*, url, content):
- """Get the package URLs for given shas by parsing the Simple API HTML.
-
- Args:
- url(str): The URL that the HTML content can be downloaded from.
- content(str): The Simple API HTML content.
-
- Returns:
- A list of structs with:
- * filename: The filename of the artifact.
- * url: The URL to download the artifact.
- * sha256: The sha256 of the artifact.
- * metadata_sha256: The whl METADATA sha256 if we can download it. If this is
- present, then the 'metadata_url' is also present. Defaults to "".
- * metadata_url: The URL for the METADATA if we can download it. Defaults to "".
- """
- sdists = {}
- whls = {}
- lines = content.split("= (2, 0):
- # We don't expect to have version 2.0 here, but have this check in place just in case.
- # https://packaging.python.org/en/latest/specifications/simple-repository-api/#versioning-pypi-s-simple-api
- fail("Unsupported API version: {}".format(api_version))
-
- for line in lines[1:]:
- dist_url, _, tail = line.partition("#sha256=")
- sha256, _, tail = tail.partition("\"")
-
- # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
- yanked = "data-yanked" in line
-
- maybe_metadata, _, tail = tail.partition(">")
- filename, _, tail = tail.partition("<")
-
- metadata_sha256 = ""
- metadata_url = ""
- for metadata_marker in ["data-core-metadata", "data-dist-info-metadata"]:
- metadata_marker = metadata_marker + "=\"sha256="
- if metadata_marker in maybe_metadata:
- # Implement https://peps.python.org/pep-0714/
- _, _, tail = maybe_metadata.partition(metadata_marker)
- metadata_sha256, _, _ = tail.partition("\"")
- metadata_url = dist_url + ".metadata"
- break
-
- if filename.endswith(".whl"):
- whls[sha256] = struct(
- filename = filename,
- url = _absolute_url(url, dist_url),
- sha256 = sha256,
- metadata_sha256 = metadata_sha256,
- metadata_url = _absolute_url(url, metadata_url),
- yanked = yanked,
- )
- else:
- sdists[sha256] = struct(
- filename = filename,
- url = _absolute_url(url, dist_url),
- sha256 = sha256,
- metadata_sha256 = "",
- metadata_url = "",
- yanked = yanked,
- )
-
- return struct(
- sdists = sdists,
- whls = whls,
- )
-
-def _absolute_url(index_url, candidate):
- if not candidate.startswith(".."):
- return candidate
-
- candidate_parts = candidate.split("..")
- last = candidate_parts[-1]
- for _ in range(len(candidate_parts) - 1):
- index_url, _, _ = index_url.rstrip("/").rpartition("/")
-
- return "{}/{}".format(index_url, last.strip("/"))
diff --git a/tests/private/pypi_index/BUILD.bazel b/tests/private/pypi_index/BUILD.bazel
deleted file mode 100644
index d365896cd3..0000000000
--- a/tests/private/pypi_index/BUILD.bazel
+++ /dev/null
@@ -1,3 +0,0 @@
-load(":pypi_index_tests.bzl", "pypi_index_test_suite")
-
-pypi_index_test_suite(name = "pypi_index_tests")
diff --git a/tests/pypi/parse_simpleapi_html/BUILD.bazel b/tests/pypi/parse_simpleapi_html/BUILD.bazel
new file mode 100644
index 0000000000..e63ef0d5fa
--- /dev/null
+++ b/tests/pypi/parse_simpleapi_html/BUILD.bazel
@@ -0,0 +1,3 @@
+load(":parse_simpleapi_html_tests.bzl", "parse_simpleapi_html_test_suite")
+
+parse_simpleapi_html_test_suite(name = "parse_simpleapi_html_tests")
diff --git a/tests/private/pypi_index/pypi_index_tests.bzl b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
similarity index 95%
rename from tests/private/pypi_index/pypi_index_tests.bzl
rename to tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
index fa381065b1..a60bb1f330 100644
--- a/tests/private/pypi_index/pypi_index_tests.bzl
+++ b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
@@ -16,7 +16,7 @@
load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("@rules_testing//lib:truth.bzl", "subjects")
-load("//python/private:pypi_index.bzl", "parse_simple_api_html") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html") # buildifier: disable=bzl-visibility
_tests = []
@@ -42,7 +42,7 @@ def _generate_html(*items):
]),
)
-def _test_parse_simple_api_html(env):
+def _test_sdist(env):
# buildifier: disable=unsorted-dict-items
tests = [
(
@@ -65,7 +65,7 @@ def _test_parse_simple_api_html(env):
for (input, want) in tests:
html = _generate_html(input)
- got = parse_simple_api_html(url = input.url, content = html)
+ got = parse_simpleapi_html(url = input.url, content = html)
env.expect.that_collection(got.sdists).has_size(1)
env.expect.that_collection(got.whls).has_size(0)
if not got:
@@ -85,9 +85,9 @@ def _test_parse_simple_api_html(env):
actual.url().equals(want.url)
actual.yanked().equals(want.yanked)
-_tests.append(_test_parse_simple_api_html)
+_tests.append(_test_sdist)
-def _test_parse_simple_api_html_whls(env):
+def _test_whls(env):
# buildifier: disable=unsorted-dict-items
tests = [
(
@@ -189,7 +189,7 @@ def _test_parse_simple_api_html_whls(env):
for (input, want) in tests:
html = _generate_html(input)
- got = parse_simple_api_html(url = input.url, content = html)
+ got = parse_simpleapi_html(url = input.url, content = html)
env.expect.that_collection(got.sdists).has_size(0)
env.expect.that_collection(got.whls).has_size(1)
if not got:
@@ -213,9 +213,9 @@ def _test_parse_simple_api_html_whls(env):
actual.url().equals(want.url)
actual.yanked().equals(want.yanked)
-_tests.append(_test_parse_simple_api_html_whls)
+_tests.append(_test_whls)
-def pypi_index_test_suite(name):
+def parse_simpleapi_html_test_suite(name):
"""Create the test suite.
Args:
From e3c660b75d3918afdb6a7665f3fc2c29aad767ca Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 14:58:53 +0900
Subject: [PATCH 12/25] move labels.bzl to private/pypi
---
python/pip_install/private/BUILD.bazel | 4 ++--
.../private/generate_group_library_build_bazel.bzl | 6 +++---
.../private/generate_whl_library_build_bazel.bzl | 6 +++---
python/private/BUILD.bazel | 5 -----
python/private/pypi/BUILD.bazel | 5 +++++
python/private/{ => pypi}/labels.bzl | 0
python/private/render_pkg_aliases.bzl | 6 +++---
7 files changed, 16 insertions(+), 16 deletions(-)
rename python/private/{ => pypi}/labels.bzl (100%)
diff --git a/python/pip_install/private/BUILD.bazel b/python/pip_install/private/BUILD.bazel
index 887d2d3468..743b174d81 100644
--- a/python/pip_install/private/BUILD.bazel
+++ b/python/pip_install/private/BUILD.bazel
@@ -28,8 +28,8 @@ bzl_library(
name = "generate_whl_library_build_bazel_bzl",
srcs = ["generate_whl_library_build_bazel.bzl"],
deps = [
- "//python/private:labels_bzl",
"//python/private:normalize_name_bzl",
+ "//python/private/pypi:labels_bzl",
],
)
@@ -37,8 +37,8 @@ bzl_library(
name = "generate_group_library_build_bazel_bzl",
srcs = ["generate_group_library_build_bazel.bzl"],
deps = [
- "//python/private:labels_bzl",
"//python/private:normalize_name_bzl",
+ "//python/private/pypi:labels_bzl",
],
)
diff --git a/python/pip_install/private/generate_group_library_build_bazel.bzl b/python/pip_install/private/generate_group_library_build_bazel.bzl
index 5fa93e22b7..7fe1fc83cd 100644
--- a/python/pip_install/private/generate_group_library_build_bazel.bzl
+++ b/python/pip_install/private/generate_group_library_build_bazel.bzl
@@ -14,15 +14,15 @@
"""Generate the BUILD.bazel contents for a repo defined by a group_library."""
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:text_util.bzl", "render")
load(
- "//python/private:labels.bzl",
+ "//python/private/pypi:labels.bzl",
"PY_LIBRARY_IMPL_LABEL",
"PY_LIBRARY_PUBLIC_LABEL",
"WHEEL_FILE_IMPL_LABEL",
"WHEEL_FILE_PUBLIC_LABEL",
)
-load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:text_util.bzl", "render")
_PRELUDE = """\
load("@rules_python//python:defs.bzl", "py_library")
diff --git a/python/pip_install/private/generate_whl_library_build_bazel.bzl b/python/pip_install/private/generate_whl_library_build_bazel.bzl
index f3ddd3bcab..72d9cf0f04 100644
--- a/python/pip_install/private/generate_whl_library_build_bazel.bzl
+++ b/python/pip_install/private/generate_whl_library_build_bazel.bzl
@@ -14,8 +14,10 @@
"""Generate the BUILD.bazel contents for a repo defined by a whl_library."""
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:text_util.bzl", "render")
load(
- "//python/private:labels.bzl",
+ "//python/private/pypi:labels.bzl",
"DATA_LABEL",
"DIST_INFO_LABEL",
"PY_LIBRARY_IMPL_LABEL",
@@ -24,8 +26,6 @@ load(
"WHEEL_FILE_IMPL_LABEL",
"WHEEL_FILE_PUBLIC_LABEL",
)
-load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:text_util.bzl", "render")
_COPY_FILE_TEMPLATE = """\
copy_file(
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index ce2c4172d3..bef3e4b103 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -291,11 +291,6 @@ bzl_library(
srcs = ["version_label.bzl"],
)
-bzl_library(
- name = "labels_bzl",
- srcs = ["labels.bzl"],
-)
-
# @bazel_tools can't define bzl_library itself, so we just put a wrapper around it.
bzl_library(
name = "bazel_tools_bzl",
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index d39af23f6b..f98c5538cc 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -48,6 +48,11 @@ bzl_library(
srcs = ["index_sources.bzl"],
)
+bzl_library(
+ name = "labels_bzl",
+ srcs = ["labels.bzl"],
+)
+
bzl_library(
name = "parse_requirements_bzl",
srcs = ["parse_requirements.bzl"],
diff --git a/python/private/labels.bzl b/python/private/pypi/labels.bzl
similarity index 100%
rename from python/private/labels.bzl
rename to python/private/pypi/labels.bzl
diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl
index 5c67d8c87f..f9491b3a00 100644
--- a/python/private/render_pkg_aliases.bzl
+++ b/python/private/render_pkg_aliases.bzl
@@ -20,10 +20,8 @@ load(
"//python/pip_install/private:generate_group_library_build_bazel.bzl",
"generate_group_library_build_bazel",
) # buildifier: disable=bzl-visibility
-load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
-load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
load(
- ":labels.bzl",
+ "//python/private/pypi:labels.bzl",
"DATA_LABEL",
"DIST_INFO_LABEL",
"PY_LIBRARY_IMPL_LABEL",
@@ -31,6 +29,8 @@ load(
"WHEEL_FILE_IMPL_LABEL",
"WHEEL_FILE_PUBLIC_LABEL",
)
+load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
load(":normalize_name.bzl", "normalize_name")
load(":text_util.bzl", "render")
From df6b857eedfec34cb6ff504efb47ac0a06aa2c39 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 15:06:42 +0900
Subject: [PATCH 13/25] move generate_build_bazel to private/pypi
---
python/pip_install/BUILD.bazel | 4 ++--
python/pip_install/pip_repository.bzl | 4 ++--
python/pip_install/private/BUILD.bazel | 18 ------------------
python/private/BUILD.bazel | 2 +-
python/private/pypi/BUILD.bazel | 18 ++++++++++++++++++
.../generate_group_library_build_bazel.bzl | 2 +-
.../pypi}/generate_whl_library_build_bazel.bzl | 2 +-
python/private/render_pkg_aliases.bzl | 2 +-
tests/pip_install/group_library/BUILD.bazel | 3 ---
tests/pip_install/whl_library/BUILD.bazel | 3 ---
.../BUILD.bazel | 3 +++
...nerate_group_library_build_bazel_tests.bzl} | 4 ++--
.../BUILD.bazel | 3 +++
...generate_whl_library_build_bazel_tests.bzl} | 4 ++--
14 files changed, 36 insertions(+), 36 deletions(-)
rename python/{pip_install/private => private/pypi}/generate_group_library_build_bazel.bzl (98%)
rename python/{pip_install/private => private/pypi}/generate_whl_library_build_bazel.bzl (99%)
delete mode 100644 tests/pip_install/group_library/BUILD.bazel
delete mode 100644 tests/pip_install/whl_library/BUILD.bazel
create mode 100644 tests/pypi/generate_group_library_build_bazel/BUILD.bazel
rename tests/{pip_install/group_library/generate_build_bazel_tests.bzl => pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl} (91%)
create mode 100644 tests/pypi/generate_whl_library_build_bazel/BUILD.bazel
rename tests/{pip_install/whl_library/generate_build_bazel_tests.bzl => pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl} (98%)
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 79b6598367..cea21e082c 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -25,8 +25,6 @@ bzl_library(
":repositories_bzl",
"//python:repositories_bzl",
"//python:versions_bzl",
- "//python/pip_install/private:generate_group_library_build_bazel_bzl",
- "//python/pip_install/private:generate_whl_library_build_bazel_bzl",
"//python/pip_install/private:srcs_bzl",
"//python/private:bzlmod_enabled_bzl",
"//python/private:envsubst_bzl",
@@ -35,6 +33,8 @@ bzl_library(
"//python/private:render_pkg_aliases_bzl",
"//python/private:repo_utils_bzl",
"//python/private:toolchains_repo_bzl",
+ "//python/private/pypi:generate_group_library_build_bazel_bzl",
+ "//python/private/pypi:generate_whl_library_build_bazel_bzl",
"//python/private/pypi:parse_requirements_bzl",
"//python/private/pypi:parse_whl_name_bzl",
"//python/private/pypi:whl_target_platforms_bzl",
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index d8a5f56193..ababfa7a56 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -18,8 +18,6 @@ load("@bazel_skylib//lib:sets.bzl", "sets")
load("//python:repositories.bzl", "is_standalone_interpreter")
load("//python:versions.bzl", "WINDOWS_NAME")
load("//python/pip_install:repositories.bzl", "all_requirements")
-load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
-load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
load("//python/private:envsubst.bzl", "envsubst")
@@ -28,6 +26,8 @@ load("//python/private:patch_whl.bzl", "patch_whl")
load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
+load("//python/private/pypi:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
+load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
diff --git a/python/pip_install/private/BUILD.bazel b/python/pip_install/private/BUILD.bazel
index 743b174d81..d0cd8cf367 100644
--- a/python/pip_install/private/BUILD.bazel
+++ b/python/pip_install/private/BUILD.bazel
@@ -24,24 +24,6 @@ srcs_module(
dest = ":srcs.bzl",
)
-bzl_library(
- name = "generate_whl_library_build_bazel_bzl",
- srcs = ["generate_whl_library_build_bazel.bzl"],
- deps = [
- "//python/private:normalize_name_bzl",
- "//python/private/pypi:labels_bzl",
- ],
-)
-
-bzl_library(
- name = "generate_group_library_build_bazel_bzl",
- srcs = ["generate_group_library_build_bazel.bzl"],
- deps = [
- "//python/private:normalize_name_bzl",
- "//python/private/pypi:labels_bzl",
- ],
-)
-
bzl_library(
name = "srcs_bzl",
srcs = ["srcs.bzl"],
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index bef3e4b103..e43561dfc5 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -243,7 +243,7 @@ bzl_library(
":normalize_name_bzl",
":text_util_bzl",
":version_label_bzl",
- "//python/pip_install/private:generate_group_library_build_bazel_bzl",
+ "//python/private/pypi:generate_group_library_build_bazel_bzl",
],
)
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index f98c5538cc..1fd40afc79 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -43,6 +43,24 @@ bzl_library(
deps = ["//python/private:enum_bzl"],
)
+bzl_library(
+ name = "generate_whl_library_build_bazel_bzl",
+ srcs = ["generate_whl_library_build_bazel.bzl"],
+ deps = [
+ ":labels_bzl",
+ "//python/private:normalize_name_bzl",
+ ],
+)
+
+bzl_library(
+ name = "generate_group_library_build_bazel_bzl",
+ srcs = ["generate_group_library_build_bazel.bzl"],
+ deps = [
+ ":labels_bzl",
+ "//python/private:normalize_name_bzl",
+ ],
+)
+
bzl_library(
name = "index_sources_bzl",
srcs = ["index_sources.bzl"],
diff --git a/python/pip_install/private/generate_group_library_build_bazel.bzl b/python/private/pypi/generate_group_library_build_bazel.bzl
similarity index 98%
rename from python/pip_install/private/generate_group_library_build_bazel.bzl
rename to python/private/pypi/generate_group_library_build_bazel.bzl
index 7fe1fc83cd..54da066b42 100644
--- a/python/pip_install/private/generate_group_library_build_bazel.bzl
+++ b/python/private/pypi/generate_group_library_build_bazel.bzl
@@ -17,7 +17,7 @@
load("//python/private:normalize_name.bzl", "normalize_name")
load("//python/private:text_util.bzl", "render")
load(
- "//python/private/pypi:labels.bzl",
+ ":labels.bzl",
"PY_LIBRARY_IMPL_LABEL",
"PY_LIBRARY_PUBLIC_LABEL",
"WHEEL_FILE_IMPL_LABEL",
diff --git a/python/pip_install/private/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl
similarity index 99%
rename from python/pip_install/private/generate_whl_library_build_bazel.bzl
rename to python/private/pypi/generate_whl_library_build_bazel.bzl
index 72d9cf0f04..d25f73a049 100644
--- a/python/pip_install/private/generate_whl_library_build_bazel.bzl
+++ b/python/private/pypi/generate_whl_library_build_bazel.bzl
@@ -17,7 +17,7 @@
load("//python/private:normalize_name.bzl", "normalize_name")
load("//python/private:text_util.bzl", "render")
load(
- "//python/private/pypi:labels.bzl",
+ ":labels.bzl",
"DATA_LABEL",
"DIST_INFO_LABEL",
"PY_LIBRARY_IMPL_LABEL",
diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl
index f9491b3a00..120c241df0 100644
--- a/python/private/render_pkg_aliases.bzl
+++ b/python/private/render_pkg_aliases.bzl
@@ -17,7 +17,7 @@
This is used in bzlmod and non-bzlmod setups."""
load(
- "//python/pip_install/private:generate_group_library_build_bazel.bzl",
+ "//python/private/pypi:generate_group_library_build_bazel.bzl",
"generate_group_library_build_bazel",
) # buildifier: disable=bzl-visibility
load(
diff --git a/tests/pip_install/group_library/BUILD.bazel b/tests/pip_install/group_library/BUILD.bazel
deleted file mode 100644
index 5a27e112db..0000000000
--- a/tests/pip_install/group_library/BUILD.bazel
+++ /dev/null
@@ -1,3 +0,0 @@
-load(":generate_build_bazel_tests.bzl", "generate_build_bazel_test_suite")
-
-generate_build_bazel_test_suite(name = "generate_build_bazel_tests")
diff --git a/tests/pip_install/whl_library/BUILD.bazel b/tests/pip_install/whl_library/BUILD.bazel
deleted file mode 100644
index 5a27e112db..0000000000
--- a/tests/pip_install/whl_library/BUILD.bazel
+++ /dev/null
@@ -1,3 +0,0 @@
-load(":generate_build_bazel_tests.bzl", "generate_build_bazel_test_suite")
-
-generate_build_bazel_test_suite(name = "generate_build_bazel_tests")
diff --git a/tests/pypi/generate_group_library_build_bazel/BUILD.bazel b/tests/pypi/generate_group_library_build_bazel/BUILD.bazel
new file mode 100644
index 0000000000..df5ab82320
--- /dev/null
+++ b/tests/pypi/generate_group_library_build_bazel/BUILD.bazel
@@ -0,0 +1,3 @@
+load(":generate_group_library_build_bazel_tests.bzl", "generate_group_library_build_bazel_test_suite")
+
+generate_group_library_build_bazel_test_suite(name = "generate_group_library_build_bazel_tests")
diff --git a/tests/pip_install/group_library/generate_build_bazel_tests.bzl b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl
similarity index 91%
rename from tests/pip_install/group_library/generate_build_bazel_tests.bzl
rename to tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl
index cf082c2990..a46aa413a3 100644
--- a/tests/pip_install/group_library/generate_build_bazel_tests.bzl
+++ b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl
@@ -15,7 +15,7 @@
""
load("@rules_testing//lib:test_suite.bzl", "test_suite")
-load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") # buildifier: disable=bzl-visibility
_tests = []
@@ -95,7 +95,7 @@ py_library(
_tests.append(_test_in_hub)
-def generate_build_bazel_test_suite(name):
+def generate_group_library_build_bazel_test_suite(name):
"""Create the test suite.
Args:
diff --git a/tests/pypi/generate_whl_library_build_bazel/BUILD.bazel b/tests/pypi/generate_whl_library_build_bazel/BUILD.bazel
new file mode 100644
index 0000000000..bea8e82ce3
--- /dev/null
+++ b/tests/pypi/generate_whl_library_build_bazel/BUILD.bazel
@@ -0,0 +1,3 @@
+load(":generate_whl_library_build_bazel_tests.bzl", "generate_whl_library_build_bazel_test_suite")
+
+generate_whl_library_build_bazel_test_suite(name = "generate_whl_library_build_bazel_tests")
diff --git a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl
similarity index 98%
rename from tests/pip_install/whl_library/generate_build_bazel_tests.bzl
rename to tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl
index 62858afc94..3d4df14b5b 100644
--- a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl
+++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl
@@ -15,7 +15,7 @@
""
load("@rules_testing//lib:test_suite.bzl", "test_suite")
-load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") # buildifier: disable=bzl-visibility
_tests = []
@@ -569,7 +569,7 @@ config_setting(
_tests.append(_test_group_member_deps_to_hub)
-def generate_build_bazel_test_suite(name):
+def generate_whl_library_build_bazel_test_suite(name):
"""Create the test suite.
Args:
From 96726d727d3fcab0e397abfe4d6b4c4cc1bca6c7 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 15:13:01 +0900
Subject: [PATCH 14/25] move render_pkg_aliases.bzl to private/pypi
---
.bazelrc | 4 ++--
python/BUILD.bazel | 2 +-
python/pip.bzl | 2 +-
python/pip_install/BUILD.bazel | 2 +-
python/pip_install/pip_repository.bzl | 2 +-
python/private/BUILD.bazel | 11 -----------
python/private/bzlmod/BUILD.bazel | 2 +-
python/private/bzlmod/pip.bzl | 2 +-
python/private/bzlmod/pip_repository.bzl | 4 ++--
python/private/pypi/BUILD.bazel | 13 +++++++++++++
python/private/{ => pypi}/render_pkg_aliases.bzl | 12 ++++++------
.../normalize_name/BUILD.bazel | 0
.../normalize_name/normalize_name_tests.bzl | 0
.../render_pkg_aliases/BUILD.bazel | 0
.../render_pkg_aliases/render_pkg_aliases_test.bzl | 4 ++--
15 files changed, 31 insertions(+), 29 deletions(-)
rename python/private/{ => pypi}/render_pkg_aliases.bzl (98%)
rename tests/{pip_hub_repository => }/normalize_name/BUILD.bazel (100%)
rename tests/{pip_hub_repository => }/normalize_name/normalize_name_tests.bzl (100%)
rename tests/{pip_hub_repository => pypi}/render_pkg_aliases/BUILD.bazel (100%)
rename tests/{pip_hub_repository => pypi}/render_pkg_aliases/render_pkg_aliases_test.bzl (99%)
diff --git a/.bazelrc b/.bazelrc
index 07188a9196..22fd7057b4 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -4,8 +4,8 @@
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
# To update these lines, execute
# `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
test --test_output=errors
diff --git a/python/BUILD.bazel b/python/BUILD.bazel
index cbf29964fb..29b495bf90 100644
--- a/python/BUILD.bazel
+++ b/python/BUILD.bazel
@@ -101,7 +101,7 @@ bzl_library(
"//python/pip_install:requirements_bzl",
"//python/private:bzlmod_enabled_bzl",
"//python/private:full_version_bzl",
- "//python/private:render_pkg_aliases_bzl",
+ "//python/private/pypi:render_pkg_aliases_bzl",
"//python/private/whl_filegroup:whl_filegroup_bzl",
],
)
diff --git a/python/pip.bzl b/python/pip.bzl
index 8cc091d3cb..fc9dbc2591 100644
--- a/python/pip.bzl
+++ b/python/pip.bzl
@@ -24,7 +24,7 @@ load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compi
load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
load("//python/private:full_version.bzl", "full_version")
load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE")
+load("//python/private/pypi:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE")
load("//python/private/whl_filegroup:whl_filegroup.bzl", _whl_filegroup = "whl_filegroup")
compile_pip_requirements = _compile_pip_requirements
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index cea21e082c..364883da4d 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -30,13 +30,13 @@ bzl_library(
"//python/private:envsubst_bzl",
"//python/private:normalize_name_bzl",
"//python/private:patch_whl_bzl",
- "//python/private:render_pkg_aliases_bzl",
"//python/private:repo_utils_bzl",
"//python/private:toolchains_repo_bzl",
"//python/private/pypi:generate_group_library_build_bazel_bzl",
"//python/private/pypi:generate_whl_library_build_bazel_bzl",
"//python/private/pypi:parse_requirements_bzl",
"//python/private/pypi:parse_whl_name_bzl",
+ "//python/private/pypi:render_pkg_aliases_bzl",
"//python/private/pypi:whl_target_platforms_bzl",
"@bazel_skylib//lib:sets",
],
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index ababfa7a56..e50c3bb17e 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -23,13 +23,13 @@ load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
load("//python/private:envsubst.bzl", "envsubst")
load("//python/private:normalize_name.bzl", "normalize_name")
load("//python/private:patch_whl.bzl", "patch_whl")
-load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
load("//python/private/pypi:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
CPPFLAGS = "CPPFLAGS"
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index e43561dfc5..68bb6c23f4 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -236,17 +236,6 @@ bzl_library(
srcs = ["register_extension_info.bzl"],
)
-bzl_library(
- name = "render_pkg_aliases_bzl",
- srcs = ["render_pkg_aliases.bzl"],
- deps = [
- ":normalize_name_bzl",
- ":text_util_bzl",
- ":version_label_bzl",
- "//python/private/pypi:generate_group_library_build_bazel_bzl",
- ],
-)
-
bzl_library(
name = "repo_utils_bzl",
srcs = ["repo_utils.bzl"],
diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel
index dfc30173dd..bae49d215a 100644
--- a/python/private/bzlmod/BUILD.bazel
+++ b/python/private/bzlmod/BUILD.bazel
@@ -54,8 +54,8 @@ bzl_library(
srcs = ["pip_repository.bzl"],
visibility = ["//:__subpackages__"],
deps = [
- "//python/private:render_pkg_aliases_bzl",
"//python/private:text_util_bzl",
+ "//python/private/pypi:render_pkg_aliases_bzl",
],
)
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index a57f42009d..c8c935c23f 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -24,11 +24,11 @@ load(
)
load("//python/private:auth.bzl", "AUTH_ATTRS")
load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:render_pkg_aliases.bzl", "whl_alias")
load("//python/private:repo_utils.bzl", "repo_utils")
load("//python/private:version_label.bzl", "version_label")
load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:render_pkg_aliases.bzl", "whl_alias")
load("//python/private/pypi:simpleapi_download.bzl", "simpleapi_download")
load("//python/private/pypi:whl_repo_name.bzl", "whl_repo_name")
load(":pip_repository.bzl", "pip_repository")
diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/bzlmod/pip_repository.bzl
index d42eb8b056..4bf3622b1b 100644
--- a/python/private/bzlmod/pip_repository.bzl
+++ b/python/private/bzlmod/pip_repository.bzl
@@ -14,12 +14,12 @@
""
+load("//python/private:text_util.bzl", "render")
load(
- "//python/private:render_pkg_aliases.bzl",
+ "//python/private/pypi:render_pkg_aliases.bzl",
"render_multiplatform_pkg_aliases",
"whl_alias",
)
-load("//python/private:text_util.bzl", "render")
_BUILD_FILE_CONTENTS = """\
package(default_visibility = ["//visibility:public"])
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 1fd40afc79..e33791448e 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -97,6 +97,19 @@ bzl_library(
srcs = ["parse_whl_name.bzl"],
)
+bzl_library(
+ name = "render_pkg_aliases_bzl",
+ srcs = ["render_pkg_aliases.bzl"],
+ deps = [
+ ":generate_group_library_build_bazel_bzl",
+ ":labels_bzl",
+ ":parse_whl_name_bzl",
+ ":whl_target_platforms_bzl",
+ "//python/private:normalize_name_bzl",
+ "//python/private:text_util_bzl",
+ ],
+)
+
bzl_library(
name = "simpleapi_download_bzl",
srcs = ["simpleapi_download.bzl"],
diff --git a/python/private/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl
similarity index 98%
rename from python/private/render_pkg_aliases.bzl
rename to python/private/pypi/render_pkg_aliases.bzl
index 120c241df0..eb907fee0f 100644
--- a/python/private/render_pkg_aliases.bzl
+++ b/python/private/pypi/render_pkg_aliases.bzl
@@ -16,12 +16,14 @@
This is used in bzlmod and non-bzlmod setups."""
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:text_util.bzl", "render")
load(
- "//python/private/pypi:generate_group_library_build_bazel.bzl",
+ ":generate_group_library_build_bazel.bzl",
"generate_group_library_build_bazel",
) # buildifier: disable=bzl-visibility
load(
- "//python/private/pypi:labels.bzl",
+ ":labels.bzl",
"DATA_LABEL",
"DIST_INFO_LABEL",
"PY_LIBRARY_IMPL_LABEL",
@@ -29,10 +31,8 @@ load(
"WHEEL_FILE_IMPL_LABEL",
"WHEEL_FILE_PUBLIC_LABEL",
)
-load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
-load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
-load(":normalize_name.bzl", "normalize_name")
-load(":text_util.bzl", "render")
+load(":parse_whl_name.bzl", "parse_whl_name")
+load(":whl_target_platforms.bzl", "whl_target_platforms")
NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\
No matching wheel for current configuration's Python version.
diff --git a/tests/pip_hub_repository/normalize_name/BUILD.bazel b/tests/normalize_name/BUILD.bazel
similarity index 100%
rename from tests/pip_hub_repository/normalize_name/BUILD.bazel
rename to tests/normalize_name/BUILD.bazel
diff --git a/tests/pip_hub_repository/normalize_name/normalize_name_tests.bzl b/tests/normalize_name/normalize_name_tests.bzl
similarity index 100%
rename from tests/pip_hub_repository/normalize_name/normalize_name_tests.bzl
rename to tests/normalize_name/normalize_name_tests.bzl
diff --git a/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel b/tests/pypi/render_pkg_aliases/BUILD.bazel
similarity index 100%
rename from tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel
rename to tests/pypi/render_pkg_aliases/BUILD.bazel
diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl
similarity index 99%
rename from tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl
rename to tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl
index 3e0ac6d609..0d4c75e3c2 100644
--- a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl
+++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl
@@ -16,8 +16,9 @@
load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility
load(
- "//python/private:render_pkg_aliases.bzl",
+ "//python/private/pypi:render_pkg_aliases.bzl",
"get_filename_config_settings",
"get_whl_flag_versions",
"multiplatform_whl_aliases",
@@ -25,7 +26,6 @@ load(
"render_pkg_aliases",
"whl_alias",
) # buildifier: disable=bzl-visibility
-load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility
def _normalize_label_strings(want):
"""normalize expected strings.
From adc19297ade5445f76622be2dd22f0b39e69330b Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 15:24:56 +0900
Subject: [PATCH 15/25] move patch_whl.bzl to private/pypi
---
python/pip_install/BUILD.bazel | 2 +-
python/pip_install/pip_repository.bzl | 3 ++-
python/private/BUILD.bazel | 6 ------
python/private/pypi/BUILD.bazel | 9 +++++++++
python/private/{ => pypi}/patch_whl.bzl | 22 +++++++---------------
python/private/{ => pypi}/repack_whl.py | 0
6 files changed, 19 insertions(+), 23 deletions(-)
rename python/private/{ => pypi}/patch_whl.bzl (87%)
rename python/private/{ => pypi}/repack_whl.py (100%)
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 364883da4d..5df2672a64 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -29,13 +29,13 @@ bzl_library(
"//python/private:bzlmod_enabled_bzl",
"//python/private:envsubst_bzl",
"//python/private:normalize_name_bzl",
- "//python/private:patch_whl_bzl",
"//python/private:repo_utils_bzl",
"//python/private:toolchains_repo_bzl",
"//python/private/pypi:generate_group_library_build_bazel_bzl",
"//python/private/pypi:generate_whl_library_build_bazel_bzl",
"//python/private/pypi:parse_requirements_bzl",
"//python/private/pypi:parse_whl_name_bzl",
+ "//python/private/pypi:patch_whl_bzl",
"//python/private/pypi:render_pkg_aliases_bzl",
"//python/private/pypi:whl_target_platforms_bzl",
"@bazel_skylib//lib:sets",
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index e50c3bb17e..fc367e3f4c 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -22,13 +22,13 @@ load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
load("//python/private:envsubst.bzl", "envsubst")
load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:patch_whl.bzl", "patch_whl")
load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
load("//python/private/pypi:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:patch_whl.bzl", "patch_whl")
load("//python/private/pypi:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
@@ -858,6 +858,7 @@ def _whl_library_impl(rctx):
whl_path = patch_whl(
rctx,
+ op = "whl_library.PatchWhl({}, {})".format(rctx.attr.name, rctx.attr.requirement),
python_interpreter = python_interpreter,
whl_path = whl_path,
patches = patches,
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 68bb6c23f4..ccc6acdcbf 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -126,12 +126,6 @@ bzl_library(
srcs = ["normalize_name.bzl"],
)
-bzl_library(
- name = "patch_whl_bzl",
- srcs = ["patch_whl.bzl"],
- deps = ["//python/private/pypi:parse_whl_name_bzl"],
-)
-
bzl_library(
name = "py_cc_toolchain_bzl",
srcs = [
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index e33791448e..7d5923c350 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -97,6 +97,15 @@ bzl_library(
srcs = ["parse_whl_name.bzl"],
)
+bzl_library(
+ name = "patch_whl_bzl",
+ srcs = ["patch_whl.bzl"],
+ deps = [
+ ":parse_whl_name_bzl",
+ "//python/private:repo_utils_bzl",
+ ],
+)
+
bzl_library(
name = "render_pkg_aliases_bzl",
srcs = ["render_pkg_aliases.bzl"],
diff --git a/python/private/patch_whl.bzl b/python/private/pypi/patch_whl.bzl
similarity index 87%
rename from python/private/patch_whl.bzl
rename to python/private/pypi/patch_whl.bzl
index 0f5d2f7727..c2c633da7f 100644
--- a/python/private/patch_whl.bzl
+++ b/python/private/pypi/patch_whl.bzl
@@ -27,7 +27,8 @@ other patches ensures that the users have overview on exactly what has changed
within the wheel.
"""
-load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private:repo_utils.bzl", "repo_utils")
+load(":parse_whl_name.bzl", "parse_whl_name")
_rules_python_root = Label("//:BUILD.bazel")
@@ -40,7 +41,7 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs):
whl_path: The whl file name to be patched.
patches: a label-keyed-int dict that has the patch files as keys and
the patch_strip as the value.
- **kwargs: extras passed to rctx.execute.
+ **kwargs: extras passed to repo_utils.execute_checked.
Returns:
value of the repackaging action.
@@ -75,11 +76,12 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs):
record_patch = rctx.path("RECORD.patch")
- result = rctx.execute(
- [
+ repo_utils.execute_checked(
+ rctx,
+ arguments = [
python_interpreter,
"-m",
- "python.private.repack_whl",
+ "python.private.pypi.repack_whl",
"--record-patch",
record_patch,
whl_input,
@@ -91,16 +93,6 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs):
**kwargs
)
- if result.return_code:
- fail(
- "repackaging .whl {whl} failed: with exit code '{return_code}':\n{stdout}\n\nstderr:\n{stderr}".format(
- whl = whl_input.basename,
- stdout = result.stdout,
- stderr = result.stderr,
- return_code = result.return_code,
- ),
- )
-
if record_patch.exists:
record_patch_contents = rctx.read(record_patch)
warning_msg = """WARNING: the resultant RECORD file of the patch wheel is different
diff --git a/python/private/repack_whl.py b/python/private/pypi/repack_whl.py
similarity index 100%
rename from python/private/repack_whl.py
rename to python/private/pypi/repack_whl.py
From 589d39f5394fe50f9fe3ad97a71af1ab38b1935c Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 15:46:35 +0900
Subject: [PATCH 16/25] Move envsubst and render_tests to top level of tests
---
tests/{private => }/envsubst/BUILD.bazel | 0
tests/{private => }/envsubst/envsubst_tests.bzl | 0
tests/{private => }/text_util/BUILD.bazel | 0
tests/{private => }/text_util/render_tests.bzl | 0
4 files changed, 0 insertions(+), 0 deletions(-)
rename tests/{private => }/envsubst/BUILD.bazel (100%)
rename tests/{private => }/envsubst/envsubst_tests.bzl (100%)
rename tests/{private => }/text_util/BUILD.bazel (100%)
rename tests/{private => }/text_util/render_tests.bzl (100%)
diff --git a/tests/private/envsubst/BUILD.bazel b/tests/envsubst/BUILD.bazel
similarity index 100%
rename from tests/private/envsubst/BUILD.bazel
rename to tests/envsubst/BUILD.bazel
diff --git a/tests/private/envsubst/envsubst_tests.bzl b/tests/envsubst/envsubst_tests.bzl
similarity index 100%
rename from tests/private/envsubst/envsubst_tests.bzl
rename to tests/envsubst/envsubst_tests.bzl
diff --git a/tests/private/text_util/BUILD.bazel b/tests/text_util/BUILD.bazel
similarity index 100%
rename from tests/private/text_util/BUILD.bazel
rename to tests/text_util/BUILD.bazel
diff --git a/tests/private/text_util/render_tests.bzl b/tests/text_util/render_tests.bzl
similarity index 100%
rename from tests/private/text_util/render_tests.bzl
rename to tests/text_util/render_tests.bzl
From 8153f102fa90fdceeca8b3e83873390cb4b69d78 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 15:47:30 +0900
Subject: [PATCH 17/25] move pip_install_srcs to private/pypi
---
python/pip_install/BUILD.bazel | 12 --------
python/pip_install/pip_repository.bzl | 4 +--
python/pip_install/private/BUILD.bazel | 30 -------------------
.../tools/dependency_resolver/BUILD.bazel | 2 +-
.../tools/wheel_installer/BUILD.bazel | 2 +-
python/private/pypi/BUILD.bazel | 17 +++++++++++
.../pypi/whl_library_utils.bzl} | 8 ++---
.../pypi/whl_library_utils_goal.bzl} | 20 ++++++-------
.../whl_library_utils}/BUILD.bazel | 10 +++----
9 files changed, 40 insertions(+), 65 deletions(-)
delete mode 100644 python/pip_install/private/BUILD.bazel
rename python/{pip_install/private/srcs.bzl => private/pypi/whl_library_utils.bzl} (77%)
rename python/{pip_install/private/pip_install_utils.bzl => private/pypi/whl_library_utils_goal.bzl} (86%)
rename tests/{pip_install => pypi/whl_library_utils}/BUILD.bazel (52%)
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 5df2672a64..715dc2865a 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -66,7 +66,6 @@ filegroup(
srcs = glob(["*.bzl"]) + [
"BUILD.bazel",
"pip_repository_requirements.bzl.tmpl",
- "//python/pip_install/private:distribution",
"//python/pip_install/tools/dependency_resolver:distribution",
"//python/pip_install/tools/wheel_installer:distribution",
],
@@ -93,17 +92,6 @@ filegroup(
visibility = ["//:__subpackages__"],
)
-filegroup(
- name = "py_srcs",
- srcs = [
- "//python/pip_install/tools/dependency_resolver:py_srcs",
- "//python/pip_install/tools/wheel_installer:py_srcs",
- "//python/private:repack_whl.py",
- "//tools:wheelmaker.py",
- ],
- visibility = ["//python/pip_install/private:__pkg__"],
-)
-
exports_files(
glob(["*.bzl"]),
visibility = ["//docs:__pkg__"],
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index fc367e3f4c..b32f393946 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -18,7 +18,6 @@ load("@bazel_skylib//lib:sets.bzl", "sets")
load("//python:repositories.bzl", "is_standalone_interpreter")
load("//python:versions.bzl", "WINDOWS_NAME")
load("//python/pip_install:repositories.bzl", "all_requirements")
-load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
load("//python/private:envsubst.bzl", "envsubst")
load("//python/private:normalize_name.bzl", "normalize_name")
@@ -30,6 +29,7 @@ load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_req
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
load("//python/private/pypi:patch_whl.bzl", "patch_whl")
load("//python/private/pypi:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
+load("//python/private/pypi:whl_library_utils.bzl", "PY_SRCS")
load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
CPPFLAGS = "CPPFLAGS"
@@ -611,7 +611,7 @@ DEPRECATED. Only left for people who vendor requirements.bzl.
"_py_srcs": attr.label_list(
doc = "Python sources used in the repository rule",
allow_files = True,
- default = PIP_INSTALL_PY_SRCS,
+ default = PY_SRCS,
),
}
diff --git a/python/pip_install/private/BUILD.bazel b/python/pip_install/private/BUILD.bazel
deleted file mode 100644
index d0cd8cf367..0000000000
--- a/python/pip_install/private/BUILD.bazel
+++ /dev/null
@@ -1,30 +0,0 @@
-load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load(":pip_install_utils.bzl", "srcs_module")
-
-package(default_visibility = ["//:__subpackages__"])
-
-exports_files([
- "srcs.bzl",
-])
-
-filegroup(
- name = "distribution",
- srcs = glob(["*"]),
- visibility = ["//python/pip_install:__subpackages__"],
-)
-
-filegroup(
- name = "bzl_srcs",
- srcs = glob(["*.bzl"]),
-)
-
-srcs_module(
- name = "srcs_module",
- srcs = "//python/pip_install:py_srcs",
- dest = ":srcs.bzl",
-)
-
-bzl_library(
- name = "srcs_bzl",
- srcs = ["srcs.bzl"],
-)
diff --git a/python/pip_install/tools/dependency_resolver/BUILD.bazel b/python/pip_install/tools/dependency_resolver/BUILD.bazel
index c2cfb39509..467b009332 100644
--- a/python/pip_install/tools/dependency_resolver/BUILD.bazel
+++ b/python/pip_install/tools/dependency_resolver/BUILD.bazel
@@ -15,5 +15,5 @@ filegroup(
include = ["**/*.py"],
exclude = ["**/*_test.py"],
),
- visibility = ["//python/pip_install:__subpackages__"],
+ visibility = ["//:__subpackages__"],
)
diff --git a/python/pip_install/tools/wheel_installer/BUILD.bazel b/python/pip_install/tools/wheel_installer/BUILD.bazel
index a396488d3d..0c24d5a489 100644
--- a/python/pip_install/tools/wheel_installer/BUILD.bazel
+++ b/python/pip_install/tools/wheel_installer/BUILD.bazel
@@ -87,5 +87,5 @@ filegroup(
include = ["**/*.py"],
exclude = ["**/*_test.py"],
),
- visibility = ["//python/pip_install:__subpackages__"],
+ visibility = ["//:__subpackages__"],
)
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 7d5923c350..1ddafad399 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -13,6 +13,7 @@
# limitations under the License.
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load(":whl_library_utils_goal.bzl", "whl_library_utils_goal")
package(default_visibility = ["//:__subpackages__"])
@@ -131,6 +132,22 @@ bzl_library(
],
)
+filegroup(
+ name = "whl_library_utils_srcs",
+ srcs = [
+ ":repack_whl.py",
+ "//python/pip_install/tools/dependency_resolver:py_srcs",
+ "//python/pip_install/tools/wheel_installer:py_srcs",
+ "//tools:wheelmaker.py",
+ ],
+)
+
+whl_library_utils_goal(
+ name = "whl_library_utils",
+ srcs = ":whl_library_utils_srcs",
+ dest = "whl_library_utils.bzl",
+)
+
bzl_library(
name = "whl_repo_name",
srcs = ["whl_repo_name.bzl"],
diff --git a/python/pip_install/private/srcs.bzl b/python/private/pypi/whl_library_utils.bzl
similarity index 77%
rename from python/pip_install/private/srcs.bzl
rename to python/private/pypi/whl_library_utils.bzl
index e92e49fc5f..7920224b95 100644
--- a/python/pip_install/private/srcs.bzl
+++ b/python/private/pypi/whl_library_utils.bzl
@@ -1,18 +1,18 @@
-"""A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
+"""A generated file containing all source files used for `@rules_python//python/private/pypi:whl_library.bzl` rules
-This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
+This file is auto-generated from the `@rules_python//tests/pypi/whl_library_utils:srcs.update` target. Please
`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
"""
# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
# sources changed.
-PIP_INSTALL_PY_SRCS = [
+PY_SRCS = [
+ "@rules_python//python/private/pypi:repack_whl.py",
"@rules_python//python/pip_install/tools/dependency_resolver:__init__.py",
"@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py",
"@rules_python//python/pip_install/tools/wheel_installer:arguments.py",
"@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py",
"@rules_python//python/pip_install/tools/wheel_installer:wheel.py",
"@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py",
- "@rules_python//python/private:repack_whl.py",
"@rules_python//tools:wheelmaker.py",
]
diff --git a/python/pip_install/private/pip_install_utils.bzl b/python/private/pypi/whl_library_utils_goal.bzl
similarity index 86%
rename from python/pip_install/private/pip_install_utils.bzl
rename to python/private/pypi/whl_library_utils_goal.bzl
index 488583dcb8..db1cc4addd 100644
--- a/python/pip_install/private/pip_install_utils.bzl
+++ b/python/private/pypi/whl_library_utils_goal.bzl
@@ -12,20 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Utilities for `rules_python` pip rules"""
+"""Utilities for `rules_python` whl_library repository rule"""
+
+load("//python/private:text_util.bzl", "render")
_SRCS_TEMPLATE = """\
-\"\"\"A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
+\"\"\"A generated file containing all source files used for `@rules_python//python/private/pypi:whl_library.bzl` rules
-This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
+This file is auto-generated from the `@rules_python//tests/pypi/whl_library_utils:srcs.update` target. Please
`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
"\"\"
# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
# sources changed.
-PIP_INSTALL_PY_SRCS = [
- {srcs}
-]
+PY_SRCS = {srcs}
"""
def _src_label(file):
@@ -45,7 +45,7 @@ def _srcs_module_impl(ctx):
ctx.actions.write(
output = output,
content = _SRCS_TEMPLATE.format(
- srcs = "\n ".join(["\"{}\",".format(src) for src in srcs]),
+ srcs = render.list(srcs),
),
)
@@ -91,12 +91,12 @@ def _srcs_updater_impl(ctx):
)
_srcs_updater = rule(
- doc = "A rule for writing a `srcs.bzl` file back to the repository",
+ doc = "A rule for writing a `whl_library_utils.bzl` file back to the repository",
implementation = _srcs_updater_impl,
attrs = {
"dest": attr.label(
doc = "The target file to write the new `input` to.",
- allow_single_file = ["srcs.bzl"],
+ allow_single_file = ["whl_library_utils.bzl"],
mandatory = True,
),
"input": attr.label(
@@ -108,7 +108,7 @@ _srcs_updater = rule(
executable = True,
)
-def srcs_module(name, dest, **kwargs):
+def whl_library_utils_goal(name, dest, **kwargs):
"""A helper rule to ensure `pip_repository` rules are always up to date
Args:
diff --git a/tests/pip_install/BUILD.bazel b/tests/pypi/whl_library_utils/BUILD.bazel
similarity index 52%
rename from tests/pip_install/BUILD.bazel
rename to tests/pypi/whl_library_utils/BUILD.bazel
index 60d25de7df..155e8afacf 100644
--- a/tests/pip_install/BUILD.bazel
+++ b/tests/pypi/whl_library_utils/BUILD.bazel
@@ -1,13 +1,13 @@
load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
diff_test(
- name = "srcs_diff_test",
+ name = "whl_library_utils_test",
failure_message = (
- "Please run 'bazel run //python/pip_install/private:srcs_module.update' " +
- "to update the 'srcs.bzl' module found in the same package."
+ "Please run 'bazel run //python/private/pypi:whl_library_utils.update' " +
+ "to update the 'whl_library_utils.bzl' module found in the same package."
),
- file1 = "//python/pip_install/private:srcs_module",
- file2 = "//python/pip_install/private:srcs.bzl",
+ file1 = "//python/private/pypi:whl_library_utils",
+ file2 = "//python/private/pypi:whl_library_utils.bzl",
# TODO: The diff_test here fails on Windows. As does the
# install script. This should be fixed.
target_compatible_with = select({
From c3525d81fec4b64c5912f29d32172e2fae65d9f5 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 17:04:00 +0900
Subject: [PATCH 18/25] split and move pip_install/pip_repository.bzl to
private/pypi
---
examples/pip_parse_vendored/requirements.bzl | 52 +-
python/pip.bzl | 3 +-
python/pip_install/BUILD.bazel | 21 +-
python/pip_install/pip_repository.bzl | 1107 +----------------
python/private/bzlmod/BUILD.bazel | 4 +-
python/private/bzlmod/pip.bzl | 18 +-
python/private/pypi/BUILD.bazel | 72 +-
python/private/pypi/attrs.bzl | 224 ++++
python/private/pypi/group_library.bzl | 40 +
python/private/pypi/package_annotation.bzl | 49 +
python/private/pypi/pip_repository.bzl | 327 +++++
python/private/pypi/pip_repository_attrs.bzl | 73 ++
.../pypi/requirements.bzl.tmpl.workspace | 72 ++
python/private/pypi/whl_library.bzl | 456 +++++++
python/private/pypi/whl_library_utils.bzl | 18 -
.../private/pypi/whl_library_utils_goal.bzl | 132 --
python/private/text_util.bzl | 3 +
tests/pypi/whl_library_utils/BUILD.bazel | 17 -
18 files changed, 1369 insertions(+), 1319 deletions(-)
create mode 100644 python/private/pypi/attrs.bzl
create mode 100644 python/private/pypi/group_library.bzl
create mode 100644 python/private/pypi/package_annotation.bzl
create mode 100644 python/private/pypi/pip_repository.bzl
create mode 100644 python/private/pypi/pip_repository_attrs.bzl
create mode 100644 python/private/pypi/requirements.bzl.tmpl.workspace
create mode 100644 python/private/pypi/whl_library.bzl
delete mode 100644 python/private/pypi/whl_library_utils.bzl
delete mode 100644 python/private/pypi/whl_library_utils_goal.bzl
delete mode 100644 tests/pypi/whl_library_utils/BUILD.bazel
diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl
index 8298f49cf2..50bfe9fe8e 100644
--- a/examples/pip_parse_vendored/requirements.bzl
+++ b/examples/pip_parse_vendored/requirements.bzl
@@ -6,16 +6,54 @@
load("@rules_python//python:pip.bzl", "pip_utils")
load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")
-all_requirements = ["@my_project_pip_deps_vendored_certifi//:pkg", "@my_project_pip_deps_vendored_charset_normalizer//:pkg", "@my_project_pip_deps_vendored_idna//:pkg", "@my_project_pip_deps_vendored_requests//:pkg", "@my_project_pip_deps_vendored_urllib3//:pkg"]
-
-all_whl_requirements_by_package = {"certifi": "@my_project_pip_deps_vendored_certifi//:whl", "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl", "idna": "@my_project_pip_deps_vendored_idna//:whl", "requests": "@my_project_pip_deps_vendored_requests//:whl", "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl"}
+all_requirements = [
+ "@my_project_pip_deps_vendored_certifi//:pkg",
+ "@my_project_pip_deps_vendored_charset_normalizer//:pkg",
+ "@my_project_pip_deps_vendored_idna//:pkg",
+ "@my_project_pip_deps_vendored_requests//:pkg",
+ "@my_project_pip_deps_vendored_urllib3//:pkg",
+]
+
+all_whl_requirements_by_package = {
+ "certifi": "@my_project_pip_deps_vendored_certifi//:whl",
+ "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl",
+ "idna": "@my_project_pip_deps_vendored_idna//:whl",
+ "requests": "@my_project_pip_deps_vendored_requests//:whl",
+ "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl",
+}
all_whl_requirements = all_whl_requirements_by_package.values()
-all_data_requirements = ["@my_project_pip_deps_vendored_certifi//:data", "@my_project_pip_deps_vendored_charset_normalizer//:data", "@my_project_pip_deps_vendored_idna//:data", "@my_project_pip_deps_vendored_requests//:data", "@my_project_pip_deps_vendored_urllib3//:data"]
-
-_packages = [("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("my_project_pip_deps_vendored_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("my_project_pip_deps_vendored_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
-_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "my_project_pip_deps_vendored", "repo_prefix": "my_project_pip_deps_vendored_", "timeout": 600}
+all_data_requirements = [
+ "@my_project_pip_deps_vendored_certifi//:data",
+ "@my_project_pip_deps_vendored_charset_normalizer//:data",
+ "@my_project_pip_deps_vendored_idna//:data",
+ "@my_project_pip_deps_vendored_requests//:data",
+ "@my_project_pip_deps_vendored_urllib3//:data",
+]
+
+_packages = [
+ ("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"),
+ ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"),
+ ("my_project_pip_deps_vendored_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"),
+ ("my_project_pip_deps_vendored_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"),
+ ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"),
+]
+_config = {
+ "download_only": False,
+ "enable_implicit_namespace_pkgs": False,
+ "environment": {},
+ "envsubst": ["PIP_RETRIES"],
+ "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"],
+ "isolated": True,
+ "pip_data_exclude": [],
+ "python_interpreter": "python3",
+ "python_interpreter_target": "@python39_host//:python",
+ "quiet": True,
+ "repo": "my_project_pip_deps_vendored",
+ "repo_prefix": "my_project_pip_deps_vendored_",
+ "timeout": 600,
+}
_annotations = {}
def requirement(name):
diff --git a/python/pip.bzl b/python/pip.bzl
index fc9dbc2591..f1c74dd964 100644
--- a/python/pip.bzl
+++ b/python/pip.bzl
@@ -19,11 +19,12 @@ symbols should not be used and they are either undocumented here or marked as
for internal use only.
"""
-load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation")
load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements")
load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
load("//python/private:full_version.bzl", "full_version")
load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation")
+load("//python/private/pypi:pip_repository.bzl", "pip_repository")
load("//python/private/pypi:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE")
load("//python/private/whl_filegroup:whl_filegroup.bzl", _whl_filegroup = "whl_filegroup")
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 715dc2865a..7cceb0fc9c 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -22,23 +22,10 @@ bzl_library(
name = "pip_repository_bzl",
srcs = ["pip_repository.bzl"],
deps = [
- ":repositories_bzl",
- "//python:repositories_bzl",
- "//python:versions_bzl",
- "//python/pip_install/private:srcs_bzl",
- "//python/private:bzlmod_enabled_bzl",
- "//python/private:envsubst_bzl",
- "//python/private:normalize_name_bzl",
- "//python/private:repo_utils_bzl",
- "//python/private:toolchains_repo_bzl",
- "//python/private/pypi:generate_group_library_build_bazel_bzl",
- "//python/private/pypi:generate_whl_library_build_bazel_bzl",
- "//python/private/pypi:parse_requirements_bzl",
- "//python/private/pypi:parse_whl_name_bzl",
- "//python/private/pypi:patch_whl_bzl",
- "//python/private/pypi:render_pkg_aliases_bzl",
- "//python/private/pypi:whl_target_platforms_bzl",
- "@bazel_skylib//lib:sets",
+ "//python/private/pypi:group_library_bzl",
+ "//python/private/pypi:package_annotation_bzl",
+ "//python/private/pypi:pip_repository_bzl",
+ "//python/private/pypi:whl_library_bzl",
],
)
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index b32f393946..18deee1993 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -14,1100 +14,13 @@
""
-load("@bazel_skylib//lib:sets.bzl", "sets")
-load("//python:repositories.bzl", "is_standalone_interpreter")
-load("//python:versions.bzl", "WINDOWS_NAME")
-load("//python/pip_install:repositories.bzl", "all_requirements")
-load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
-load("//python/private:envsubst.bzl", "envsubst")
-load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
-load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
-load("//python/private/pypi:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
-load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
-load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
-load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
-load("//python/private/pypi:patch_whl.bzl", "patch_whl")
-load("//python/private/pypi:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
-load("//python/private/pypi:whl_library_utils.bzl", "PY_SRCS")
-load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms")
-
-CPPFLAGS = "CPPFLAGS"
-
-COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools"
-
-_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point"
-
-def _construct_pypath(rctx):
- """Helper function to construct a PYTHONPATH.
-
- Contains entries for code in this repo as well as packages downloaded from //python/pip_install:repositories.bzl.
- This allows us to run python code inside repository rule implementations.
-
- Args:
- rctx: Handle to the repository_context.
-
- Returns: String of the PYTHONPATH.
- """
-
- separator = ":" if not "windows" in rctx.os.name.lower() else ";"
- pypath = separator.join([
- str(rctx.path(entry).dirname)
- for entry in rctx.attr._python_path_entries
- ])
- return pypath
-
-def _get_python_interpreter_attr(rctx):
- """A helper function for getting the `python_interpreter` attribute or it's default
-
- Args:
- rctx (repository_ctx): Handle to the rule repository context.
-
- Returns:
- str: The attribute value or it's default
- """
- if rctx.attr.python_interpreter:
- return rctx.attr.python_interpreter
-
- if "win" in rctx.os.name:
- return "python.exe"
- else:
- return "python3"
-
-def _resolve_python_interpreter(rctx):
- """Helper function to find the python interpreter from the common attributes
-
- Args:
- rctx: Handle to the rule repository context.
-
- Returns:
- `path` object, for the resolved path to the Python interpreter.
- """
- python_interpreter = _get_python_interpreter_attr(rctx)
-
- if rctx.attr.python_interpreter_target != None:
- python_interpreter = rctx.path(rctx.attr.python_interpreter_target)
-
- (os, _) = get_host_os_arch(rctx)
-
- # On Windows, the symlink doesn't work because Windows attempts to find
- # Python DLLs where the symlink is, not where the symlink points.
- if os == WINDOWS_NAME:
- python_interpreter = python_interpreter.realpath
- elif "/" not in python_interpreter:
- # It's a plain command, e.g. "python3", to look up in the environment.
- found_python_interpreter = rctx.which(python_interpreter)
- if not found_python_interpreter:
- fail("python interpreter `{}` not found in PATH".format(python_interpreter))
- python_interpreter = found_python_interpreter
- else:
- python_interpreter = rctx.path(python_interpreter)
- return python_interpreter
-
-def _get_xcode_location_cflags(rctx):
- """Query the xcode sdk location to update cflags
-
- Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so.
- Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
- otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
- """
-
- # Only run on MacOS hosts
- if not rctx.os.name.lower().startswith("mac os"):
- return []
-
- xcode_sdk_location = repo_utils.execute_unchecked(
- rctx,
- op = "GetXcodeLocation",
- arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"],
- )
- if xcode_sdk_location.return_code != 0:
- return []
-
- xcode_root = xcode_sdk_location.stdout.strip()
- if COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower():
- # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer
- # so we need to change the path to to the macos specific tools which are in a different relative
- # path than xcode installed command line tools.
- xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root)
- return [
- "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root),
- ]
-
-def _get_toolchain_unix_cflags(rctx, python_interpreter):
- """Gather cflags from a standalone toolchain for unix systems.
-
- Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
- otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
- """
-
- # Only run on Unix systems
- if not rctx.os.name.lower().startswith(("mac os", "linux")):
- return []
-
- # Only update the location when using a standalone toolchain.
- if not is_standalone_interpreter(rctx, python_interpreter):
- return []
-
- stdout = repo_utils.execute_checked_stdout(
- rctx,
- op = "GetPythonVersionForUnixCflags",
- arguments = [
- python_interpreter,
- "-c",
- "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')",
- ],
- )
- _python_version = stdout
- include_path = "{}/include/python{}".format(
- python_interpreter.dirname,
- _python_version,
- )
-
- return ["-isystem {}".format(include_path)]
-
-def use_isolated(ctx, attr):
- """Determine whether or not to pass the pip `--isolated` flag to the pip invocation.
-
- Args:
- ctx: repository or module context
- attr: attributes for the repo rule or tag extension
-
- Returns:
- True if --isolated should be passed
- """
- use_isolated = attr.isolated
-
- # The environment variable will take precedence over the attribute
- isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None)
- if isolated_env != None:
- if isolated_env.lower() in ("0", "false"):
- use_isolated = False
- else:
- use_isolated = True
-
- return use_isolated
-
-def _parse_optional_attrs(rctx, args, extra_pip_args = None):
- """Helper function to parse common attributes of pip_repository and whl_library repository rules.
-
- This function also serializes the structured arguments as JSON
- so they can be passed on the command line to subprocesses.
-
- Args:
- rctx: Handle to the rule repository context.
- args: A list of parsed args for the rule.
- extra_pip_args: The pip args to pass.
- Returns: Augmented args list.
- """
-
- if use_isolated(rctx, rctx.attr):
- args.append("--isolated")
-
- # Bazel version 7.1.0 and later (and rolling releases from version 8.0.0-pre.20240128.3)
- # support rctx.getenv(name, default): When building incrementally, any change to the value of
- # the variable named by name will cause this repository to be re-fetched.
- if "getenv" in dir(rctx):
- getenv = rctx.getenv
- else:
- getenv = rctx.os.environ.get
-
- # Check for None so we use empty default types from our attrs.
- # Some args want to be list, and some want to be dict.
- if extra_pip_args != None:
- args += [
- "--extra_pip_args",
- json.encode(struct(arg = [
- envsubst(pip_arg, rctx.attr.envsubst, getenv)
- for pip_arg in rctx.attr.extra_pip_args
- ])),
- ]
-
- if rctx.attr.download_only:
- args.append("--download_only")
-
- if rctx.attr.pip_data_exclude != None:
- args += [
- "--pip_data_exclude",
- json.encode(struct(arg = rctx.attr.pip_data_exclude)),
- ]
-
- if rctx.attr.enable_implicit_namespace_pkgs:
- args.append("--enable_implicit_namespace_pkgs")
-
- if rctx.attr.environment != None:
- args += [
- "--environment",
- json.encode(struct(arg = rctx.attr.environment)),
- ]
-
- return args
-
-def _create_repository_execution_environment(rctx, python_interpreter):
- """Create a environment dictionary for processes we spawn with rctx.execute.
-
- Args:
- rctx (repository_ctx): The repository context.
- python_interpreter (path): The resolved python interpreter.
- Returns:
- Dictionary of environment variable suitable to pass to rctx.execute.
- """
-
- # Gather any available CPPFLAGS values
- cppflags = []
- cppflags.extend(_get_xcode_location_cflags(rctx))
- cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter))
-
- env = {
- "PYTHONPATH": _construct_pypath(rctx),
- CPPFLAGS: " ".join(cppflags),
- }
-
- return env
-
-_BUILD_FILE_CONTENTS = """\
-package(default_visibility = ["//visibility:public"])
-
-# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
-exports_files(["requirements.bzl"])
-"""
-
-def _pip_repository_impl(rctx):
- requirements_by_platform = parse_requirements(
- rctx,
- requirements_by_platform = rctx.attr.requirements_by_platform,
- requirements_linux = rctx.attr.requirements_linux,
- requirements_lock = rctx.attr.requirements_lock,
- requirements_osx = rctx.attr.requirements_darwin,
- requirements_windows = rctx.attr.requirements_windows,
- extra_pip_args = rctx.attr.extra_pip_args,
- )
- selected_requirements = {}
- options = None
- repository_platform = host_platform(rctx.os)
- for name, requirements in requirements_by_platform.items():
- r = select_requirement(
- requirements,
- platform = repository_platform,
- )
- if not r:
- continue
- options = options or r.extra_pip_args
- selected_requirements[name] = r.requirement_line
-
- bzl_packages = sorted(selected_requirements.keys())
-
- # Normalize cycles first
- requirement_cycles = {
- name: sorted(sets.to_list(sets.make(deps)))
- for name, deps in rctx.attr.experimental_requirement_cycles.items()
- }
-
- # Check for conflicts between cycles _before_ we normalize package names so
- # that reported errors use the names the user specified
- for i in range(len(requirement_cycles)):
- left_group = requirement_cycles.keys()[i]
- left_deps = requirement_cycles.values()[i]
- for j in range(len(requirement_cycles) - (i + 1)):
- right_deps = requirement_cycles.values()[1 + i + j]
- right_group = requirement_cycles.keys()[1 + i + j]
- for d in left_deps:
- if d in right_deps:
- fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group))
-
- # And normalize the names as used in the cycle specs
- #
- # NOTE: We must check that a listed dependency is actually in the actual
- # requirements set for the current platform so that we can support cycles in
- # platform-conditional requirements. Otherwise we'll blindly generate a
- # label referencing a package which may not be installed on the current
- # platform.
- requirement_cycles = {
- normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages])
- for name, group in requirement_cycles.items()
- }
-
- imports = [
- # NOTE: Maintain the order consistent with `buildifier`
- 'load("@rules_python//python:pip.bzl", "pip_utils")',
- 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")',
- ]
-
- annotations = {}
- for pkg, annotation in rctx.attr.annotations.items():
- filename = "{}.annotation.json".format(normalize_name(pkg))
- rctx.file(filename, json.encode_indent(json.decode(annotation)))
- annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename)
-
- config = {
- "download_only": rctx.attr.download_only,
- "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs,
- "environment": rctx.attr.environment,
- "envsubst": rctx.attr.envsubst,
- "extra_pip_args": options,
- "isolated": use_isolated(rctx, rctx.attr),
- "pip_data_exclude": rctx.attr.pip_data_exclude,
- "python_interpreter": _get_python_interpreter_attr(rctx),
- "quiet": rctx.attr.quiet,
- "repo": rctx.attr.name,
- "timeout": rctx.attr.timeout,
- }
- if rctx.attr.use_hub_alias_dependencies:
- config["dep_template"] = "@{}//{{name}}:{{target}}".format(rctx.attr.name)
- else:
- config["repo_prefix"] = "{}_".format(rctx.attr.name)
-
- if rctx.attr.python_interpreter_target:
- config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target)
- if rctx.attr.experimental_target_platforms:
- config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms
-
- macro_tmpl = "@%s//{}:{}" % rctx.attr.name
-
- aliases = render_pkg_aliases(
- aliases = {
- pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)]
- for pkg in bzl_packages or []
- },
- )
- for path, contents in aliases.items():
- rctx.file(path, contents)
-
- rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
- rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
- " # %%GROUP_LIBRARY%%": """\
- group_repo = "{name}__groups"
- group_library(
- name = group_repo,
- repo_prefix = "{name}_",
- groups = all_requirement_groups,
- )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "",
- "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([
- macro_tmpl.format(p, "data")
- for p in bzl_packages
- ]),
- "%%ALL_REQUIREMENTS%%": _format_repr_list([
- macro_tmpl.format(p, "pkg")
- for p in bzl_packages
- ]),
- "%%ALL_REQUIREMENT_GROUPS%%": _format_dict(_repr_dict(requirement_cycles)),
- "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": _format_dict(_repr_dict({
- p: macro_tmpl.format(p, "whl")
- for p in bzl_packages
- })),
- "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)),
- "%%CONFIG%%": _format_dict(_repr_dict(config)),
- "%%EXTRA_PIP_ARGS%%": json.encode(options),
- "%%IMPORTS%%": "\n".join(imports),
- "%%MACRO_TMPL%%": macro_tmpl,
- "%%NAME%%": rctx.attr.name,
- "%%PACKAGES%%": _format_repr_list(
- [
- ("{}_{}".format(rctx.attr.name, p), r)
- for p, r in sorted(selected_requirements.items())
- ],
- ),
- })
-
- return
-
-common_env = [
- "RULES_PYTHON_PIP_ISOLATED",
- REPO_DEBUG_ENV_VAR,
-]
-
-common_attrs = {
- "download_only": attr.bool(
- doc = """
-Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of
---platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different
-platform from the host platform.
- """,
- ),
- "enable_implicit_namespace_pkgs": attr.bool(
- default = False,
- doc = """
-If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary
-and py_test targets must specify either `legacy_create_init=False` or the global Bazel option
-`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory.
-
-This option is required to support some packages which cannot handle the conversion to pkg-util style.
- """,
- ),
- "environment": attr.string_dict(
- doc = """
-Environment variables to set in the pip subprocess.
-Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy`
-Note that pip is run with "--isolated" on the CLI so `PIP__`
-style env vars are ignored, but env vars that control requests and urllib3
-can be passed. If you need `PIP__`, take a look at `extra_pip_args`
-and `envsubst`.
- """,
- default = {},
- ),
- "envsubst": attr.string_list(
- mandatory = False,
- doc = """\
-A list of environment variables to substitute (e.g. `["PIP_INDEX_URL",
-"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args`
-using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset)
-or `${VARNAME:-default}` (expanding to default if the variable is unset or empty
-in the environment). Note: On Bazel 6 and Bazel 7.0 changes to the variables named
-here do not cause packages to be re-fetched. Don't fetch different things based
-on the value of these variables.
-""",
- ),
- "experimental_requirement_cycles": attr.string_list_dict(
- default = {},
- doc = """\
-A mapping of dependency cycle names to a list of requirements which form that cycle.
-
-Requirements which form cycles will be installed together and taken as
-dependencies together in order to ensure that the cycle is always satisified.
-
-Example:
- `sphinx` depends on `sphinxcontrib-serializinghtml`
- When listing both as requirements, ala
-
- ```
- py_binary(
- name = "doctool",
- ...
- deps = [
- "@pypi//sphinx:pkg",
- "@pypi//sphinxcontrib_serializinghtml",
- ]
- )
- ```
-
- Will produce a Bazel error such as
-
- ```
- ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph:
- //:doctool (...)
- @pypi//sphinxcontrib_serializinghtml:pkg (...)
- .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...)
- | @pypi_sphinxcontrib_serializinghtml//:_pkg (...)
- | @pypi_sphinx//:pkg (...)
- | @pypi_sphinx//:_pkg (...)
- `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...)
- ```
-
- Which we can resolve by configuring these two requirements to be installed together as a cycle
-
- ```
- pip_parse(
- ...
- experimental_requirement_cycles = {
- "sphinx": [
- "sphinx",
- "sphinxcontrib-serializinghtml",
- ]
- },
- )
- ```
-
-Warning:
- If a dependency participates in multiple cycles, all of those cycles must be
- collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed
- as two separate cycles.
-""",
- ),
- "experimental_target_platforms": attr.string_list(
- default = [],
- doc = """\
-A list of platforms that we will generate the conditional dependency graph for
-cross platform wheels by parsing the wheel metadata. This will generate the
-correct dependencies for packages like `sphinx` or `pylint`, which include
-`colorama` when installed and used on Windows platforms.
-
-An empty list means falling back to the legacy behaviour where the host
-platform is the target platform.
-
-WARNING: It may not work as expected in cases where the python interpreter
-implementation that is being used at runtime is different between different platforms.
-This has been tested for CPython only.
-
-For specific target platforms use values of the form `_` where ``
-is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`,
-`aarch64`, `s390x` and `ppc64le`.
-
-You can also target a specific Python version by using `cp3__`.
-If multiple python versions are specified as target platforms, then select statements
-of the `lib` and `whl` targets will include usage of version aware toolchain config
-settings like `@rules_python//python/config_settings:is_python_3.y`.
-
-Special values: `host` (for generating deps for the host platform only) and
-`_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`.
-
-NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly.
-""",
- ),
- "extra_pip_args": attr.string_list(
- doc = """Extra arguments to pass on to pip. Must not contain spaces.
-
-Supports environment variables using the syntax `$VARNAME` or
-`${VARNAME}` (expanding to empty string if unset) or
-`${VARNAME:-default}` (expanding to default if the variable is unset
-or empty in the environment), if `"VARNAME"` is listed in the
-`envsubst` attribute. See also `envsubst`.
-""",
- ),
- "isolated": attr.bool(
- doc = """\
-Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to
-the underlying pip command. Alternatively, the `RULES_PYTHON_PIP_ISOLATED` environment variable can be used
-to control this flag.
-""",
- default = True,
- ),
- "pip_data_exclude": attr.string_list(
- doc = "Additional data exclusion parameters to add to the pip packages BUILD file.",
- ),
- "python_interpreter": attr.string(
- doc = """\
-The python interpreter to use. This can either be an absolute path or the name
-of a binary found on the host's `PATH` environment variable. If no value is set
-`python3` is defaulted for Unix systems and `python.exe` for Windows.
-""",
- # NOTE: This attribute should not have a default. See `_get_python_interpreter_attr`
- # default = "python3"
- ),
- "python_interpreter_target": attr.label(
- allow_single_file = True,
- doc = """
-If you are using a custom python interpreter built by another repository rule,
-use this attribute to specify its BUILD target. This allows pip_repository to invoke
-pip using the same interpreter as your toolchain. If set, takes precedence over
-python_interpreter. An example value: "@python3_x86_64-unknown-linux-gnu//:python".
-""",
- ),
- "quiet": attr.bool(
- default = True,
- doc = """\
-If True, suppress printing stdout and stderr output to the terminal.
-
-If you would like to get more diagnostic output, please use:
-
- RULES_PYTHON_REPO_DEBUG=1
-
-or
-
- RULES_PYTHON_REPO_DEBUG_VERBOSITY=
-""",
- ),
- "repo_prefix": attr.string(
- doc = """
-Prefix for the generated packages will be of the form `@//...`
-
-DEPRECATED. Only left for people who vendor requirements.bzl.
-""",
- ),
- # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute
- "timeout": attr.int(
- default = 600,
- doc = "Timeout (in seconds) on the rule's execution duration.",
- ),
- "_py_srcs": attr.label_list(
- doc = "Python sources used in the repository rule",
- allow_files = True,
- default = PY_SRCS,
- ),
-}
-
-pip_repository_attrs = {
- "annotations": attr.string_dict(
- doc = "Optional annotations to apply to packages",
- ),
- "requirements_by_platform": attr.label_keyed_string_dict(
- doc = """\
-The requirements files and the comma delimited list of target platforms as values.
-
-The keys are the requirement files and the values are comma-separated platform
-identifiers. For now we only support `_` values that are present in
-`@platforms//os` and `@platforms//cpu` packages respectively.
-""",
- ),
- "requirements_darwin": attr.label(
- allow_single_file = True,
- doc = "Override the requirements_lock attribute when the host platform is Mac OS",
- ),
- "requirements_linux": attr.label(
- allow_single_file = True,
- doc = "Override the requirements_lock attribute when the host platform is Linux",
- ),
- "requirements_lock": attr.label(
- allow_single_file = True,
- doc = """\
-A fully resolved 'requirements.txt' pip requirement file containing the
-transitive set of your dependencies. If this file is passed instead of
-'requirements' no resolve will take place and pip_repository will create
-individual repositories for each of your dependencies so that wheels are
-fetched/built only for the targets specified by 'build/run/test'. Note that if
-your lockfile is platform-dependent, you can use the `requirements_[platform]`
-attributes.
-
-Note, that in general requirements files are compiled for a specific platform,
-but sometimes they can work for multiple platforms. `rules_python` right now
-supports requirements files that are created for a particular platform without
-platform markers.
-""",
- ),
- "requirements_windows": attr.label(
- allow_single_file = True,
- doc = "Override the requirements_lock attribute when the host platform is Windows",
- ),
- "use_hub_alias_dependencies": attr.bool(
- default = False,
- doc = """\
-Controls if the hub alias dependencies are used. If set to true, then the
-group_library will be included in the hub repo.
-
-True will become default in a subsequent release.
-""",
- ),
- "_template": attr.label(
- default = ":pip_repository_requirements.bzl.tmpl",
- ),
-}
-
-pip_repository_attrs.update(**common_attrs)
-
-pip_repository = repository_rule(
- attrs = pip_repository_attrs,
- doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within.
-
-Those dependencies become available in a generated `requirements.bzl` file.
-You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below.
-
-In your WORKSPACE file:
-
-```starlark
-load("@rules_python//python:pip.bzl", "pip_parse")
-
-pip_parse(
- name = "pypi",
- requirements_lock = ":requirements.txt",
-)
-
-load("@pypi//:requirements.bzl", "install_deps")
-
-install_deps()
-```
-
-You can then reference installed dependencies from a `BUILD` file with the alias targets generated in the same repo, for example, for `PyYAML` we would have the following:
-- `@pypi//pyyaml` and `@pypi//pyyaml:pkg` both point to the `py_library`
- created after extracting the `PyYAML` package.
-- `@pypi//pyyaml:data` points to the extra data included in the package.
-- `@pypi//pyyaml:dist_info` points to the `dist-info` files in the package.
-- `@pypi//pyyaml:whl` points to the wheel file that was extracted.
-
-```starlark
-py_library(
- name = "bar",
- ...
- deps = [
- "//my/other:dep",
- "@pypi//numpy",
- "@pypi//requests",
- ],
-)
-```
-
-or
-
-```starlark
-load("@pypi//:requirements.bzl", "requirement")
-
-py_library(
- name = "bar",
- ...
- deps = [
- "//my/other:dep",
- requirement("numpy"),
- requirement("requests"),
- ],
-)
-```
-
-In addition to the `requirement` macro, which is used to access the generated `py_library`
-target generated from a package's wheel, The generated `requirements.bzl` file contains
-functionality for exposing [entry points][whl_ep] as `py_binary` targets as well.
-
-[whl_ep]: https://packaging.python.org/specifications/entry-points/
-
-```starlark
-load("@pypi//:requirements.bzl", "entry_point")
-
-alias(
- name = "pip-compile",
- actual = entry_point(
- pkg = "pip-tools",
- script = "pip-compile",
- ),
-)
-```
-
-Note that for packages whose name and script are the same, only the name of the package
-is needed when calling the `entry_point` macro.
-
-```starlark
-load("@pip//:requirements.bzl", "entry_point")
-
-alias(
- name = "flake8",
- actual = entry_point("flake8"),
-)
-```
-
-### Vendoring the requirements.bzl file
-
-In some cases you may not want to generate the requirements.bzl file as a repository rule
-while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module
-such as a ruleset, you may want to include the requirements.bzl file rather than make your users
-install the WORKSPACE setup to generate it.
-See https://github.com/bazelbuild/rules_python/issues/608
-
-This is the same workflow as Gazelle, which creates `go_repository` rules with
-[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos)
-
-To do this, use the "write to source file" pattern documented in
-https://blog.aspect.dev/bazel-can-write-to-the-source-folder
-to put a copy of the generated requirements.bzl into your project.
-Then load the requirements.bzl file directly rather than from the generated repository.
-See the example in rules_python/examples/pip_parse_vendored.
-""",
- implementation = _pip_repository_impl,
- environ = common_env,
-)
-
-def _whl_library_impl(rctx):
- python_interpreter = _resolve_python_interpreter(rctx)
- args = [
- python_interpreter,
- "-m",
- "python.pip_install.tools.wheel_installer.wheel_installer",
- "--requirement",
- rctx.attr.requirement,
- ]
- extra_pip_args = []
- extra_pip_args.extend(rctx.attr.extra_pip_args)
-
- # Manually construct the PYTHONPATH since we cannot use the toolchain here
- environment = _create_repository_execution_environment(rctx, python_interpreter)
-
- whl_path = None
- if rctx.attr.whl_file:
- whl_path = rctx.path(rctx.attr.whl_file)
-
- # Simulate the behaviour where the whl is present in the current directory.
- rctx.symlink(whl_path, whl_path.basename)
- whl_path = rctx.path(whl_path.basename)
- elif rctx.attr.urls:
- filename = rctx.attr.filename
- urls = rctx.attr.urls
- if not filename:
- _, _, filename = urls[0].rpartition("/")
-
- if not (filename.endswith(".whl") or filename.endswith("tar.gz") or filename.endswith(".zip")):
- if rctx.attr.filename:
- msg = "got '{}'".format(filename)
- else:
- msg = "detected '{}' from url:\n{}".format(filename, urls[0])
- fail("Only '.whl', '.tar.gz' or '.zip' files are supported, {}".format(msg))
-
- result = rctx.download(
- url = urls,
- output = filename,
- sha256 = rctx.attr.sha256,
- auth = get_auth(rctx, urls),
- )
-
- if not result.success:
- fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))
-
- if filename.endswith(".whl"):
- whl_path = rctx.path(rctx.attr.filename)
- else:
- # It is an sdist and we need to tell PyPI to use a file in this directory
- # and not use any indexes.
- extra_pip_args.extend(["--no-index", "--find-links", "."])
-
- args = _parse_optional_attrs(rctx, args, extra_pip_args)
-
- if not whl_path:
- repo_utils.execute_checked(
- rctx,
- op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement),
- arguments = args,
- environment = environment,
- quiet = rctx.attr.quiet,
- timeout = rctx.attr.timeout,
- )
-
- whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
- if not rctx.delete("whl_file.json"):
- fail("failed to delete the whl_file.json file")
-
- if rctx.attr.whl_patches:
- patches = {}
- for patch_file, json_args in rctx.attr.whl_patches.items():
- patch_dst = struct(**json.decode(json_args))
- if whl_path.basename in patch_dst.whls:
- patches[patch_file] = patch_dst.patch_strip
-
- whl_path = patch_whl(
- rctx,
- op = "whl_library.PatchWhl({}, {})".format(rctx.attr.name, rctx.attr.requirement),
- python_interpreter = python_interpreter,
- whl_path = whl_path,
- patches = patches,
- quiet = rctx.attr.quiet,
- timeout = rctx.attr.timeout,
- )
-
- target_platforms = rctx.attr.experimental_target_platforms
- if target_platforms:
- parsed_whl = parse_whl_name(whl_path.basename)
- if parsed_whl.platform_tag != "any":
- # NOTE @aignas 2023-12-04: if the wheel is a platform specific
- # wheel, we only include deps for that target platform
- target_platforms = [
- p.target_platform
- for p in whl_target_platforms(
- platform_tag = parsed_whl.platform_tag,
- abi_tag = parsed_whl.abi_tag,
- )
- ]
-
- repo_utils.execute_checked(
- rctx,
- op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
- arguments = args + [
- "--whl-file",
- whl_path,
- ] + ["--platform={}".format(p) for p in target_platforms],
- environment = environment,
- quiet = rctx.attr.quiet,
- timeout = rctx.attr.timeout,
- )
-
- metadata = json.decode(rctx.read("metadata.json"))
- rctx.delete("metadata.json")
-
- entry_points = {}
- for item in metadata["entry_points"]:
- name = item["name"]
- module = item["module"]
- attribute = item["attribute"]
-
- # There is an extreme edge-case with entry_points that end with `.py`
- # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
- entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name
- entry_point_target_name = (
- _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py
- )
- entry_point_script_name = entry_point_target_name + ".py"
-
- rctx.file(
- entry_point_script_name,
- _generate_entry_point_contents(module, attribute),
- )
- entry_points[entry_point_without_py] = entry_point_script_name
-
- build_file_contents = generate_whl_library_build_bazel(
- dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
- whl_name = whl_path.basename,
- dependencies = metadata["deps"],
- dependencies_by_platform = metadata["deps_by_platform"],
- group_name = rctx.attr.group_name,
- group_deps = rctx.attr.group_deps,
- data_exclude = rctx.attr.pip_data_exclude,
- tags = [
- "pypi_name=" + metadata["name"],
- "pypi_version=" + metadata["version"],
- ],
- entry_points = entry_points,
- annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
- )
- rctx.file("BUILD.bazel", build_file_contents)
-
- return
-
-def _generate_entry_point_contents(
- module,
- attribute,
- shebang = "#!/usr/bin/env python3"):
- """Generate the contents of an entry point script.
-
- Args:
- module (str): The name of the module to use.
- attribute (str): The name of the attribute to call.
- shebang (str, optional): The shebang to use for the entry point python
- file.
-
- Returns:
- str: A string of python code.
- """
- contents = """\
-{shebang}
-import sys
-from {module} import {attribute}
-if __name__ == "__main__":
- sys.exit({attribute}())
-""".format(
- shebang = shebang,
- module = module,
- attribute = attribute,
- )
- return contents
-
-# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique
-whl_library_attrs = dict({
- "annotation": attr.label(
- doc = (
- "Optional json encoded file containing annotation to apply to the extracted wheel. " +
- "See `package_annotation`"
- ),
- allow_files = True,
- ),
- "dep_template": attr.string(
- doc = """
-The dep template to use for referencing the dependencies. It should have `{name}`
-and `{target}` tokens that will be replaced with the normalized distribution name
-and the target that we need respectively.
-""",
- ),
- "filename": attr.string(
- doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.",
- ),
- "group_deps": attr.string_list(
- doc = "List of dependencies to skip in order to break the cycles within a dependency group.",
- default = [],
- ),
- "group_name": attr.string(
- doc = "Name of the group, if any.",
- ),
- "repo": attr.string(
- mandatory = True,
- doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.",
- ),
- "requirement": attr.string(
- mandatory = True,
- doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.",
- ),
- "sha256": attr.string(
- doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.",
- ),
- "urls": attr.string_list(
- doc = """\
-The list of urls of the whl to be downloaded using bazel downloader. Using this
-attr makes `extra_pip_args` and `download_only` ignored.""",
- ),
- "whl_file": attr.label(
- doc = "The whl file that should be used instead of downloading or building the whl.",
- ),
- "whl_patches": attr.label_keyed_string_dict(
- doc = """a label-keyed-string dict that has
- json.encode(struct([whl_file], patch_strip]) as values. This
- is to maintain flexibility and correct bzlmod extension interface
- until we have a better way to define whl_library and move whl
- patching to a separate place. INTERNAL USE ONLY.""",
- ),
- "_python_path_entries": attr.label_list(
- # Get the root directory of these rules and keep them as a default attribute
- # in order to avoid unnecessary repository fetching restarts.
- #
- # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478
- default = [
- Label("//:BUILD.bazel"),
- ] + [
- # Includes all the external dependencies from repositories.bzl
- Label("@" + repo + "//:BUILD.bazel")
- for repo in all_requirements
- ],
- ),
-}, **common_attrs)
-whl_library_attrs.update(AUTH_ATTRS)
-
-whl_library = repository_rule(
- attrs = whl_library_attrs,
- doc = """
-Download and extracts a single wheel based into a bazel repo based on the requirement string passed in.
-Instantiated from pip_repository and inherits config options from there.""",
- implementation = _whl_library_impl,
- environ = common_env,
-)
-
-def package_annotation(
- additive_build_content = None,
- copy_files = {},
- copy_executables = {},
- data = [],
- data_exclude_glob = [],
- srcs_exclude_glob = []):
- """Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule.
-
- [cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md
-
- Args:
- additive_build_content (str, optional): Raw text to add to the generated `BUILD` file of a package.
- copy_files (dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf]
- copy_executables (dict, optional): A mapping of `src` and `out` files for
- [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
- executable.
- data (list, optional): A list of labels to add as `data` dependencies to the generated `py_library` target.
- data_exclude_glob (list, optional): A list of exclude glob patterns to add as `data` to the generated
- `py_library` target.
- srcs_exclude_glob (list, optional): A list of labels to add as `srcs` to the generated `py_library` target.
-
- Returns:
- str: A json encoded string of the provided content.
- """
- return json.encode(struct(
- additive_build_content = additive_build_content,
- copy_files = copy_files,
- copy_executables = copy_executables,
- data = data,
- data_exclude_glob = data_exclude_glob,
- srcs_exclude_glob = srcs_exclude_glob,
- ))
-
-def _group_library_impl(rctx):
- build_file_contents = generate_group_library_build_bazel(
- repo_prefix = rctx.attr.repo_prefix,
- groups = rctx.attr.groups,
- )
- rctx.file("BUILD.bazel", build_file_contents)
-
-group_library = repository_rule(
- attrs = {
- "groups": attr.string_list_dict(
- doc = "A mapping of group names to requirements within that group.",
- ),
- "repo_prefix": attr.string(
- doc = "Prefix used for the whl_library created components of each group",
- ),
- },
- implementation = _group_library_impl,
- doc = """
-Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups.
-This is an implementation detail of dependency groups and should not be used alone.
- """,
-)
-
-# pip_repository implementation
-
-def _format_list(items):
- return "[{}]".format(", ".join(items))
-
-def _format_repr_list(strings):
- return _format_list(
- [repr(s) for s in strings],
- )
-
-def _repr_dict(items):
- return {k: repr(v) for k, v in items.items()}
-
-def _format_dict(items):
- return "{{{}}}".format(", ".join(sorted(['"{}": {}'.format(k, v) for k, v in items.items()])))
+load("//python/private/pypi:group_library.bzl", _group_library = "group_library")
+load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation")
+load("//python/private/pypi:pip_repository.bzl", _pip_repository = "pip_repository")
+load("//python/private/pypi:whl_library.bzl", _whl_library = "whl_library")
+
+# Re-exports for backwards compatibility
+group_library = _group_library
+pip_repository = _pip_repository
+whl_library = _whl_library
+package_annotation = _package_annotation
diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel
index bae49d215a..b76e905998 100644
--- a/python/private/bzlmod/BUILD.bazel
+++ b/python/private/bzlmod/BUILD.bazel
@@ -30,7 +30,9 @@ bzl_library(
srcs = ["pip.bzl"],
deps = [
":pip_repository_bzl",
- "//python/pip_install:pip_repository_bzl",
+ "//python/private/pypi:pip_repository_attrs_bzl",
+ "//python/private/pypi:whl_library_bzl",
+ "//python/private/pypi:attrs_bzl",
"//python/private/pypi:simpleapi_download_bzl",
"//python/private:full_version_bzl",
"//python/private:normalize_name_bzl",
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index c8c935c23f..187a48e7a2 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -16,20 +16,17 @@
load("@bazel_features//:features.bzl", "bazel_features")
load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS")
-load(
- "//python/pip_install:pip_repository.bzl",
- "pip_repository_attrs",
- "use_isolated",
- "whl_library",
-)
load("//python/private:auth.bzl", "AUTH_ATTRS")
load("//python/private:normalize_name.bzl", "normalize_name")
load("//python/private:repo_utils.bzl", "repo_utils")
load("//python/private:version_label.bzl", "version_label")
+load("//python/private/pypi:attrs.bzl", "use_isolated")
load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private/pypi:pip_repository_attrs.bzl", "ATTRS")
load("//python/private/pypi:render_pkg_aliases.bzl", "whl_alias")
load("//python/private/pypi:simpleapi_download.bzl", "simpleapi_download")
+load("//python/private/pypi:whl_library.bzl", "whl_library")
load("//python/private/pypi:whl_repo_name.bzl", "whl_repo_name")
load(":pip_repository.bzl", "pip_repository")
@@ -618,16 +615,9 @@ A dict of labels to wheel names that is typically generated by the whl_modificat
The labels are JSON config files describing the modifications.
""",
),
- }, **pip_repository_attrs)
+ }, **ATTRS)
attrs.update(AUTH_ATTRS)
- # Like the pip_repository rule, we end up setting this manually so
- # don't allow users to override it.
- attrs.pop("repo_prefix")
-
- # annotations has been replaced with whl_modifications in bzlmod
- attrs.pop("annotations")
-
return attrs
def _whl_mod_attrs():
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 1ddafad399..79f72f0089 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -13,7 +13,6 @@
# limitations under the License.
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load(":whl_library_utils_goal.bzl", "whl_library_utils_goal")
package(default_visibility = ["//:__subpackages__"])
@@ -32,6 +31,13 @@ filegroup(
visibility = ["//python/private:__pkg__"],
)
+# Keep sorted by library name and keep the files named by the main symbol they export
+
+bzl_library(
+ name = "attrs_bzl",
+ srcs = ["attrs.bzl"],
+)
+
bzl_library(
name = "config_settings_bzl",
srcs = ["config_settings.bzl"],
@@ -62,6 +68,14 @@ bzl_library(
],
)
+bzl_library(
+ name = "group_library_bzl",
+ srcs = ["group_library.bzl"],
+ deps = [
+ ":generate_group_library_build_bazel_bzl",
+ ],
+)
+
bzl_library(
name = "index_sources_bzl",
srcs = ["index_sources.bzl"],
@@ -72,6 +86,11 @@ bzl_library(
srcs = ["labels.bzl"],
)
+bzl_library(
+ name = "package_annotation_bzl",
+ srcs = ["package_annotation.bzl"],
+)
+
bzl_library(
name = "parse_requirements_bzl",
srcs = ["parse_requirements.bzl"],
@@ -107,6 +126,26 @@ bzl_library(
],
)
+bzl_library(
+ name = "pip_repository_bzl",
+ srcs = ["pip_repository.bzl"],
+ deps = [
+ ":attrs_bzl",
+ ":parse_requirements_bzl",
+ ":pip_repository_attrs_bzl",
+ ":render_pkg_aliases_bzl",
+ "//python/private:normalize_name_bzl",
+ "//python/private:repo_utils_bzl",
+ "//python/private:text_util_bzl",
+ "@bazel_skylib//lib:sets",
+ ],
+)
+
+bzl_library(
+ name = "pip_repository_attrs_bzl",
+ srcs = ["pip_repository_attrs.bzl"],
+)
+
bzl_library(
name = "render_pkg_aliases_bzl",
srcs = ["render_pkg_aliases.bzl"],
@@ -132,24 +171,27 @@ bzl_library(
],
)
-filegroup(
- name = "whl_library_utils_srcs",
- srcs = [
- ":repack_whl.py",
- "//python/pip_install/tools/dependency_resolver:py_srcs",
- "//python/pip_install/tools/wheel_installer:py_srcs",
- "//tools:wheelmaker.py",
+bzl_library(
+ name = "whl_library_bzl",
+ srcs = ["whl_library.bzl"],
+ deps = [
+ ":attrs_bzl",
+ ":generate_whl_library_build_bazel_bzl",
+ ":parse_whl_name_bzl",
+ ":patch_whl_bzl",
+ ":whl_target_platforms_bzl",
+ "//python:repositories_bzl",
+ "//python:versions_bzl",
+ "//python/pip_install:repositories_bzl",
+ "//python/private:auth_bzl",
+ "//python/private:envsubst_bzl",
+ "//python/private:repo_utils_bzl",
+ "//python/private:toolchains_repo_bzl",
],
)
-whl_library_utils_goal(
- name = "whl_library_utils",
- srcs = ":whl_library_utils_srcs",
- dest = "whl_library_utils.bzl",
-)
-
bzl_library(
- name = "whl_repo_name",
+ name = "whl_repo_name_bzl",
srcs = ["whl_repo_name.bzl"],
deps = [
":parse_whl_name_bzl",
diff --git a/python/private/pypi/attrs.bzl b/python/private/pypi/attrs.bzl
new file mode 100644
index 0000000000..79ffea54a1
--- /dev/null
+++ b/python/private/pypi/attrs.bzl
@@ -0,0 +1,224 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"common attributes for whl_library and pip_repository"
+
+ATTRS = {
+ "download_only": attr.bool(
+ doc = """
+Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of
+--platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different
+platform from the host platform.
+ """,
+ ),
+ "enable_implicit_namespace_pkgs": attr.bool(
+ default = False,
+ doc = """
+If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary
+and py_test targets must specify either `legacy_create_init=False` or the global Bazel option
+`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory.
+
+This option is required to support some packages which cannot handle the conversion to pkg-util style.
+ """,
+ ),
+ "environment": attr.string_dict(
+ doc = """
+Environment variables to set in the pip subprocess.
+Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy`
+Note that pip is run with "--isolated" on the CLI so `PIP__`
+style env vars are ignored, but env vars that control requests and urllib3
+can be passed. If you need `PIP__`, take a look at `extra_pip_args`
+and `envsubst`.
+ """,
+ default = {},
+ ),
+ "envsubst": attr.string_list(
+ mandatory = False,
+ doc = """\
+A list of environment variables to substitute (e.g. `["PIP_INDEX_URL",
+"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args`
+using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset)
+or `${VARNAME:-default}` (expanding to default if the variable is unset or empty
+in the environment). Note: On Bazel 6 and Bazel 7.0 changes to the variables named
+here do not cause packages to be re-fetched. Don't fetch different things based
+on the value of these variables.
+""",
+ ),
+ "experimental_requirement_cycles": attr.string_list_dict(
+ default = {},
+ doc = """\
+A mapping of dependency cycle names to a list of requirements which form that cycle.
+
+Requirements which form cycles will be installed together and taken as
+dependencies together in order to ensure that the cycle is always satisified.
+
+Example:
+ `sphinx` depends on `sphinxcontrib-serializinghtml`
+ When listing both as requirements, ala
+
+ ```
+ py_binary(
+ name = "doctool",
+ ...
+ deps = [
+ "@pypi//sphinx:pkg",
+ "@pypi//sphinxcontrib_serializinghtml",
+ ]
+ )
+ ```
+
+ Will produce a Bazel error such as
+
+ ```
+ ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph:
+ //:doctool (...)
+ @pypi//sphinxcontrib_serializinghtml:pkg (...)
+ .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...)
+ | @pypi_sphinxcontrib_serializinghtml//:_pkg (...)
+ | @pypi_sphinx//:pkg (...)
+ | @pypi_sphinx//:_pkg (...)
+ `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...)
+ ```
+
+ Which we can resolve by configuring these two requirements to be installed together as a cycle
+
+ ```
+ pip_parse(
+ ...
+ experimental_requirement_cycles = {
+ "sphinx": [
+ "sphinx",
+ "sphinxcontrib-serializinghtml",
+ ]
+ },
+ )
+ ```
+
+Warning:
+ If a dependency participates in multiple cycles, all of those cycles must be
+ collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed
+ as two separate cycles.
+""",
+ ),
+ "experimental_target_platforms": attr.string_list(
+ default = [],
+ doc = """\
+A list of platforms that we will generate the conditional dependency graph for
+cross platform wheels by parsing the wheel metadata. This will generate the
+correct dependencies for packages like `sphinx` or `pylint`, which include
+`colorama` when installed and used on Windows platforms.
+
+An empty list means falling back to the legacy behaviour where the host
+platform is the target platform.
+
+WARNING: It may not work as expected in cases where the python interpreter
+implementation that is being used at runtime is different between different platforms.
+This has been tested for CPython only.
+
+For specific target platforms use values of the form `_` where ``
+is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`,
+`aarch64`, `s390x` and `ppc64le`.
+
+You can also target a specific Python version by using `cp3__`.
+If multiple python versions are specified as target platforms, then select statements
+of the `lib` and `whl` targets will include usage of version aware toolchain config
+settings like `@rules_python//python/config_settings:is_python_3.y`.
+
+Special values: `host` (for generating deps for the host platform only) and
+`_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`.
+
+NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly.
+""",
+ ),
+ "extra_pip_args": attr.string_list(
+ doc = """Extra arguments to pass on to pip. Must not contain spaces.
+
+Supports environment variables using the syntax `$VARNAME` or
+`${VARNAME}` (expanding to empty string if unset) or
+`${VARNAME:-default}` (expanding to default if the variable is unset
+or empty in the environment), if `"VARNAME"` is listed in the
+`envsubst` attribute. See also `envsubst`.
+""",
+ ),
+ "isolated": attr.bool(
+ doc = """\
+Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to
+the underlying pip command. Alternatively, the `RULES_PYTHON_PIP_ISOLATED` environment variable can be used
+to control this flag.
+""",
+ default = True,
+ ),
+ "pip_data_exclude": attr.string_list(
+ doc = "Additional data exclusion parameters to add to the pip packages BUILD file.",
+ ),
+ "python_interpreter": attr.string(
+ doc = """\
+The python interpreter to use. This can either be an absolute path or the name
+of a binary found on the host's `PATH` environment variable. If no value is set
+`python3` is defaulted for Unix systems and `python.exe` for Windows.
+""",
+ # NOTE: This attribute should not have a default. See `_get_python_interpreter_attr`
+ # default = "python3"
+ ),
+ "python_interpreter_target": attr.label(
+ allow_single_file = True,
+ doc = """
+If you are using a custom python interpreter built by another repository rule,
+use this attribute to specify its BUILD target. This allows pip_repository to invoke
+pip using the same interpreter as your toolchain. If set, takes precedence over
+python_interpreter. An example value: "@python3_x86_64-unknown-linux-gnu//:python".
+""",
+ ),
+ "quiet": attr.bool(
+ default = True,
+ doc = """\
+If True, suppress printing stdout and stderr output to the terminal.
+
+If you would like to get more diagnostic output, please use:
+
+ RULES_PYTHON_REPO_DEBUG=1
+
+or
+
+ RULES_PYTHON_REPO_DEBUG_VERBOSITY=
+""",
+ ),
+ # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute
+ "timeout": attr.int(
+ default = 600,
+ doc = "Timeout (in seconds) on the rule's execution duration.",
+ ),
+}
+
+def use_isolated(ctx, attr):
+ """Determine whether or not to pass the pip `--isolated` flag to the pip invocation.
+
+ Args:
+ ctx: repository or module context
+ attr: attributes for the repo rule or tag extension
+
+ Returns:
+ True if --isolated should be passed
+ """
+ use_isolated = attr.isolated
+
+ # The environment variable will take precedence over the attribute
+ isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None)
+ if isolated_env != None:
+ if isolated_env.lower() in ("0", "false"):
+ use_isolated = False
+ else:
+ use_isolated = True
+
+ return use_isolated
diff --git a/python/private/pypi/group_library.bzl b/python/private/pypi/group_library.bzl
new file mode 100644
index 0000000000..ff800e2f18
--- /dev/null
+++ b/python/private/pypi/group_library.bzl
@@ -0,0 +1,40 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""group_library implementation for WORKSPACE setups."""
+
+load(":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
+
+def _group_library_impl(rctx):
+ build_file_contents = generate_group_library_build_bazel(
+ repo_prefix = rctx.attr.repo_prefix,
+ groups = rctx.attr.groups,
+ )
+ rctx.file("BUILD.bazel", build_file_contents)
+
+group_library = repository_rule(
+ attrs = {
+ "groups": attr.string_list_dict(
+ doc = "A mapping of group names to requirements within that group.",
+ ),
+ "repo_prefix": attr.string(
+ doc = "Prefix used for the whl_library created components of each group",
+ ),
+ },
+ implementation = _group_library_impl,
+ doc = """
+Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups.
+This is an implementation detail of dependency groups and should not be used alone.
+ """,
+)
diff --git a/python/private/pypi/package_annotation.bzl b/python/private/pypi/package_annotation.bzl
new file mode 100644
index 0000000000..4a54703ac4
--- /dev/null
+++ b/python/private/pypi/package_annotation.bzl
@@ -0,0 +1,49 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package annotation API for WORKSPACE setups."""
+
+def package_annotation(
+ additive_build_content = None,
+ copy_files = {},
+ copy_executables = {},
+ data = [],
+ data_exclude_glob = [],
+ srcs_exclude_glob = []):
+ """Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule.
+
+ [cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md
+
+ Args:
+ additive_build_content (str, optional): Raw text to add to the generated `BUILD` file of a package.
+ copy_files (dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf]
+ copy_executables (dict, optional): A mapping of `src` and `out` files for
+ [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
+ executable.
+ data (list, optional): A list of labels to add as `data` dependencies to the generated `py_library` target.
+ data_exclude_glob (list, optional): A list of exclude glob patterns to add as `data` to the generated
+ `py_library` target.
+ srcs_exclude_glob (list, optional): A list of labels to add as `srcs` to the generated `py_library` target.
+
+ Returns:
+ str: A json encoded string of the provided content.
+ """
+ return json.encode(struct(
+ additive_build_content = additive_build_content,
+ copy_files = copy_files,
+ copy_executables = copy_executables,
+ data = data,
+ data_exclude_glob = data_exclude_glob,
+ srcs_exclude_glob = srcs_exclude_glob,
+ ))
diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl
new file mode 100644
index 0000000000..ab537606b1
--- /dev/null
+++ b/python/private/pypi/pip_repository.bzl
@@ -0,0 +1,327 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""
+
+load("@bazel_skylib//lib:sets.bzl", "sets")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR")
+load("//python/private:text_util.bzl", "render")
+load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
+load(":pip_repository_attrs.bzl", "ATTRS")
+load(":render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
+
+def _get_python_interpreter_attr(rctx):
+ """A helper function for getting the `python_interpreter` attribute or it's default
+
+ Args:
+ rctx (repository_ctx): Handle to the rule repository context.
+
+ Returns:
+ str: The attribute value or it's default
+ """
+ if rctx.attr.python_interpreter:
+ return rctx.attr.python_interpreter
+
+ if "win" in rctx.os.name:
+ return "python.exe"
+ else:
+ return "python3"
+
+def use_isolated(ctx, attr):
+ """Determine whether or not to pass the pip `--isolated` flag to the pip invocation.
+
+ Args:
+ ctx: repository or module context
+ attr: attributes for the repo rule or tag extension
+
+ Returns:
+ True if --isolated should be passed
+ """
+ use_isolated = attr.isolated
+
+ # The environment variable will take precedence over the attribute
+ isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None)
+ if isolated_env != None:
+ if isolated_env.lower() in ("0", "false"):
+ use_isolated = False
+ else:
+ use_isolated = True
+
+ return use_isolated
+
+_BUILD_FILE_CONTENTS = """\
+package(default_visibility = ["//visibility:public"])
+
+# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
+exports_files(["requirements.bzl"])
+"""
+
+def _pip_repository_impl(rctx):
+ requirements_by_platform = parse_requirements(
+ rctx,
+ requirements_by_platform = rctx.attr.requirements_by_platform,
+ requirements_linux = rctx.attr.requirements_linux,
+ requirements_lock = rctx.attr.requirements_lock,
+ requirements_osx = rctx.attr.requirements_darwin,
+ requirements_windows = rctx.attr.requirements_windows,
+ extra_pip_args = rctx.attr.extra_pip_args,
+ )
+ selected_requirements = {}
+ options = None
+ repository_platform = host_platform(rctx.os)
+ for name, requirements in requirements_by_platform.items():
+ r = select_requirement(
+ requirements,
+ platform = repository_platform,
+ )
+ if not r:
+ continue
+ options = options or r.extra_pip_args
+ selected_requirements[name] = r.requirement_line
+
+ bzl_packages = sorted(selected_requirements.keys())
+
+ # Normalize cycles first
+ requirement_cycles = {
+ name: sorted(sets.to_list(sets.make(deps)))
+ for name, deps in rctx.attr.experimental_requirement_cycles.items()
+ }
+
+ # Check for conflicts between cycles _before_ we normalize package names so
+ # that reported errors use the names the user specified
+ for i in range(len(requirement_cycles)):
+ left_group = requirement_cycles.keys()[i]
+ left_deps = requirement_cycles.values()[i]
+ for j in range(len(requirement_cycles) - (i + 1)):
+ right_deps = requirement_cycles.values()[1 + i + j]
+ right_group = requirement_cycles.keys()[1 + i + j]
+ for d in left_deps:
+ if d in right_deps:
+ fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group))
+
+ # And normalize the names as used in the cycle specs
+ #
+ # NOTE: We must check that a listed dependency is actually in the actual
+ # requirements set for the current platform so that we can support cycles in
+ # platform-conditional requirements. Otherwise we'll blindly generate a
+ # label referencing a package which may not be installed on the current
+ # platform.
+ requirement_cycles = {
+ normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages])
+ for name, group in requirement_cycles.items()
+ }
+
+ imports = [
+ # NOTE: Maintain the order consistent with `buildifier`
+ 'load("@rules_python//python:pip.bzl", "pip_utils")',
+ 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")',
+ ]
+
+ annotations = {}
+ for pkg, annotation in rctx.attr.annotations.items():
+ filename = "{}.annotation.json".format(normalize_name(pkg))
+ rctx.file(filename, json.encode_indent(json.decode(annotation)))
+ annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename)
+
+ config = {
+ "download_only": rctx.attr.download_only,
+ "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs,
+ "environment": rctx.attr.environment,
+ "envsubst": rctx.attr.envsubst,
+ "extra_pip_args": options,
+ "isolated": use_isolated(rctx, rctx.attr),
+ "pip_data_exclude": rctx.attr.pip_data_exclude,
+ "python_interpreter": _get_python_interpreter_attr(rctx),
+ "quiet": rctx.attr.quiet,
+ "repo": rctx.attr.name,
+ "timeout": rctx.attr.timeout,
+ }
+ if rctx.attr.use_hub_alias_dependencies:
+ config["dep_template"] = "@{}//{{name}}:{{target}}".format(rctx.attr.name)
+ else:
+ config["repo_prefix"] = "{}_".format(rctx.attr.name)
+
+ if rctx.attr.python_interpreter_target:
+ config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target)
+ if rctx.attr.experimental_target_platforms:
+ config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms
+
+ macro_tmpl = "@%s//{}:{}" % rctx.attr.name
+
+ aliases = render_pkg_aliases(
+ aliases = {
+ pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)]
+ for pkg in bzl_packages or []
+ },
+ )
+ for path, contents in aliases.items():
+ rctx.file(path, contents)
+
+ rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
+ rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
+ " # %%GROUP_LIBRARY%%": """\
+ group_repo = "{name}__groups"
+ group_library(
+ name = group_repo,
+ repo_prefix = "{name}_",
+ groups = all_requirement_groups,
+ )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "",
+ "%%ALL_DATA_REQUIREMENTS%%": render.list([
+ macro_tmpl.format(p, "data")
+ for p in bzl_packages
+ ]),
+ "%%ALL_REQUIREMENTS%%": render.list([
+ macro_tmpl.format(p, "pkg")
+ for p in bzl_packages
+ ]),
+ "%%ALL_REQUIREMENT_GROUPS%%": render.dict(requirement_cycles),
+ "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({
+ p: macro_tmpl.format(p, "whl")
+ for p in bzl_packages
+ }),
+ "%%ANNOTATIONS%%": render.dict(annotations),
+ "%%CONFIG%%": render.dict(config),
+ "%%EXTRA_PIP_ARGS%%": json.encode(options),
+ "%%IMPORTS%%": "\n".join(imports),
+ "%%MACRO_TMPL%%": macro_tmpl,
+ "%%NAME%%": rctx.attr.name,
+ "%%PACKAGES%%": render.list(
+ [
+ ("{}_{}".format(rctx.attr.name, p), r)
+ for p, r in sorted(selected_requirements.items())
+ ],
+ ),
+ })
+
+ return
+
+pip_repository = repository_rule(
+ attrs = dict(
+ annotations = attr.string_dict(
+ doc = "Optional annotations to apply to packages",
+ ),
+ _template = attr.label(
+ default = ":requirements.bzl.tmpl.workspace",
+ ),
+ **ATTRS
+ ),
+ doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within.
+
+Those dependencies become available in a generated `requirements.bzl` file.
+You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below.
+
+In your WORKSPACE file:
+
+```starlark
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+pip_parse(
+ name = "pypi",
+ requirements_lock = ":requirements.txt",
+)
+
+load("@pypi//:requirements.bzl", "install_deps")
+
+install_deps()
+```
+
+You can then reference installed dependencies from a `BUILD` file with the alias targets generated in the same repo, for example, for `PyYAML` we would have the following:
+- `@pypi//pyyaml` and `@pypi//pyyaml:pkg` both point to the `py_library`
+ created after extracting the `PyYAML` package.
+- `@pypi//pyyaml:data` points to the extra data included in the package.
+- `@pypi//pyyaml:dist_info` points to the `dist-info` files in the package.
+- `@pypi//pyyaml:whl` points to the wheel file that was extracted.
+
+```starlark
+py_library(
+ name = "bar",
+ ...
+ deps = [
+ "//my/other:dep",
+ "@pypi//numpy",
+ "@pypi//requests",
+ ],
+)
+```
+
+or
+
+```starlark
+load("@pypi//:requirements.bzl", "requirement")
+
+py_library(
+ name = "bar",
+ ...
+ deps = [
+ "//my/other:dep",
+ requirement("numpy"),
+ requirement("requests"),
+ ],
+)
+```
+
+In addition to the `requirement` macro, which is used to access the generated `py_library`
+target generated from a package's wheel, The generated `requirements.bzl` file contains
+functionality for exposing [entry points][whl_ep] as `py_binary` targets as well.
+
+[whl_ep]: https://packaging.python.org/specifications/entry-points/
+
+```starlark
+load("@pypi//:requirements.bzl", "entry_point")
+
+alias(
+ name = "pip-compile",
+ actual = entry_point(
+ pkg = "pip-tools",
+ script = "pip-compile",
+ ),
+)
+```
+
+Note that for packages whose name and script are the same, only the name of the package
+is needed when calling the `entry_point` macro.
+
+```starlark
+load("@pip//:requirements.bzl", "entry_point")
+
+alias(
+ name = "flake8",
+ actual = entry_point("flake8"),
+)
+```
+
+### Vendoring the requirements.bzl file
+
+In some cases you may not want to generate the requirements.bzl file as a repository rule
+while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module
+such as a ruleset, you may want to include the requirements.bzl file rather than make your users
+install the WORKSPACE setup to generate it.
+See https://github.com/bazelbuild/rules_python/issues/608
+
+This is the same workflow as Gazelle, which creates `go_repository` rules with
+[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos)
+
+To do this, use the "write to source file" pattern documented in
+https://blog.aspect.dev/bazel-can-write-to-the-source-folder
+to put a copy of the generated requirements.bzl into your project.
+Then load the requirements.bzl file directly rather than from the generated repository.
+See the example in rules_python/examples/pip_parse_vendored.
+""",
+ implementation = _pip_repository_impl,
+ environ = [
+ "RULES_PYTHON_PIP_ISOLATED",
+ REPO_DEBUG_ENV_VAR,
+ ],
+)
diff --git a/python/private/pypi/pip_repository_attrs.bzl b/python/private/pypi/pip_repository_attrs.bzl
new file mode 100644
index 0000000000..23000869e9
--- /dev/null
+++ b/python/private/pypi/pip_repository_attrs.bzl
@@ -0,0 +1,73 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Common attributes between bzlmod pip.parse and workspace pip_parse.
+
+A common attributes shared between bzlmod and workspace implementations
+stored in a separate file to avoid unnecessary refetching of the
+repositories."""
+
+load(":attrs.bzl", COMMON_ATTRS = "ATTRS")
+
+ATTRS = {
+ "requirements_by_platform": attr.label_keyed_string_dict(
+ doc = """\
+The requirements files and the comma delimited list of target platforms as values.
+
+The keys are the requirement files and the values are comma-separated platform
+identifiers. For now we only support `_` values that are present in
+`@platforms//os` and `@platforms//cpu` packages respectively.
+""",
+ ),
+ "requirements_darwin": attr.label(
+ allow_single_file = True,
+ doc = "Override the requirements_lock attribute when the host platform is Mac OS",
+ ),
+ "requirements_linux": attr.label(
+ allow_single_file = True,
+ doc = "Override the requirements_lock attribute when the host platform is Linux",
+ ),
+ "requirements_lock": attr.label(
+ allow_single_file = True,
+ doc = """\
+A fully resolved 'requirements.txt' pip requirement file containing the
+transitive set of your dependencies. If this file is passed instead of
+'requirements' no resolve will take place and pip_repository will create
+individual repositories for each of your dependencies so that wheels are
+fetched/built only for the targets specified by 'build/run/test'. Note that if
+your lockfile is platform-dependent, you can use the `requirements_[platform]`
+attributes.
+
+Note, that in general requirements files are compiled for a specific platform,
+but sometimes they can work for multiple platforms. `rules_python` right now
+supports requirements files that are created for a particular platform without
+platform markers.
+""",
+ ),
+ "requirements_windows": attr.label(
+ allow_single_file = True,
+ doc = "Override the requirements_lock attribute when the host platform is Windows",
+ ),
+ "use_hub_alias_dependencies": attr.bool(
+ default = False,
+ doc = """\
+Controls if the hub alias dependencies are used. If set to true, then the
+group_library will be included in the hub repo.
+
+True will become default in a subsequent release.
+""",
+ ),
+}
+
+ATTRS.update(**COMMON_ATTRS)
diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace
new file mode 100644
index 0000000000..2f4bcd6916
--- /dev/null
+++ b/python/private/pypi/requirements.bzl.tmpl.workspace
@@ -0,0 +1,72 @@
+"""Starlark representation of locked requirements.
+
+@generated by rules_python pip_parse repository rule.
+"""
+
+%%IMPORTS%%
+
+all_requirements = %%ALL_REQUIREMENTS%%
+
+all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%
+
+all_whl_requirements = all_whl_requirements_by_package.values()
+
+all_data_requirements = %%ALL_DATA_REQUIREMENTS%%
+
+_packages = %%PACKAGES%%
+_config = %%CONFIG%%
+_annotations = %%ANNOTATIONS%%
+
+def requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg")
+
+def whl_requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl")
+
+def data_requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "data")
+
+def dist_info_requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info")
+
+def _get_annotation(requirement):
+ # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11`
+ # down to `setuptools`.
+ name = requirement.split(" ")[0].split("=")[0].split("[")[0]
+ return _annotations.get(name)
+
+def install_deps(**whl_library_kwargs):
+ """Repository rule macro. Install dependencies from `pip_parse`.
+
+ Args:
+ **whl_library_kwargs: Additional arguments which will flow to underlying
+ `whl_library` calls. See pip_repository.bzl for details.
+ """
+
+ # Set up the requirement groups
+ all_requirement_groups = %%ALL_REQUIREMENT_GROUPS%%
+
+ requirement_group_mapping = {
+ requirement: group_name
+ for group_name, group_requirements in all_requirement_groups.items()
+ for requirement in group_requirements
+ }
+
+ # %%GROUP_LIBRARY%%
+
+ # Install wheels which may be participants in a group
+ whl_config = dict(_config)
+ whl_config.update(whl_library_kwargs)
+
+ for name, requirement in _packages:
+ group_name = requirement_group_mapping.get(name.replace("%%NAME%%_", ""))
+ group_deps = all_requirement_groups.get(group_name, [])
+
+ whl_library(
+ name = name,
+ requirement = requirement,
+ group_name = group_name,
+ group_deps = group_deps,
+ annotation = _get_annotation(requirement),
+ **whl_config
+ )
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
new file mode 100644
index 0000000000..d365b52154
--- /dev/null
+++ b/python/private/pypi/whl_library.bzl
@@ -0,0 +1,456 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""
+
+load("//python:repositories.bzl", "is_standalone_interpreter")
+load("//python:versions.bzl", "WINDOWS_NAME")
+load("//python/pip_install:repositories.bzl", "all_requirements")
+load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
+load("//python/private:envsubst.bzl", "envsubst")
+load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
+load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
+load(":attrs.bzl", "ATTRS", "use_isolated")
+load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
+load(":parse_whl_name.bzl", "parse_whl_name")
+load(":patch_whl.bzl", "patch_whl")
+load(":whl_target_platforms.bzl", "whl_target_platforms")
+
+_CPPFLAGS = "CPPFLAGS"
+_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools"
+
+def _construct_pypath(rctx):
+ """Helper function to construct a PYTHONPATH.
+
+ Contains entries for code in this repo as well as packages downloaded from //python/pip_install:repositories.bzl.
+ This allows us to run python code inside repository rule implementations.
+
+ Args:
+ rctx: Handle to the repository_context.
+
+ Returns: String of the PYTHONPATH.
+ """
+
+ separator = ":" if not "windows" in rctx.os.name.lower() else ";"
+ pypath = separator.join([
+ str(rctx.path(entry).dirname)
+ for entry in rctx.attr._python_path_entries
+ ])
+ return pypath
+
+def _get_python_interpreter_attr(rctx):
+ """A helper function for getting the `python_interpreter` attribute or it's default
+
+ Args:
+ rctx (repository_ctx): Handle to the rule repository context.
+
+ Returns:
+ str: The attribute value or it's default
+ """
+ if rctx.attr.python_interpreter:
+ return rctx.attr.python_interpreter
+
+ if "win" in rctx.os.name:
+ return "python.exe"
+ else:
+ return "python3"
+
+def _resolve_python_interpreter(rctx):
+ """Helper function to find the python interpreter from the common attributes
+
+ Args:
+ rctx: Handle to the rule repository context.
+
+ Returns:
+ `path` object, for the resolved path to the Python interpreter.
+ """
+ python_interpreter = _get_python_interpreter_attr(rctx)
+
+ if rctx.attr.python_interpreter_target != None:
+ python_interpreter = rctx.path(rctx.attr.python_interpreter_target)
+
+ (os, _) = get_host_os_arch(rctx)
+
+ # On Windows, the symlink doesn't work because Windows attempts to find
+ # Python DLLs where the symlink is, not where the symlink points.
+ if os == WINDOWS_NAME:
+ python_interpreter = python_interpreter.realpath
+ elif "/" not in python_interpreter:
+ # It's a plain command, e.g. "python3", to look up in the environment.
+ found_python_interpreter = rctx.which(python_interpreter)
+ if not found_python_interpreter:
+ fail("python interpreter `{}` not found in PATH".format(python_interpreter))
+ python_interpreter = found_python_interpreter
+ else:
+ python_interpreter = rctx.path(python_interpreter)
+ return python_interpreter
+
+def _get_xcode_location_cflags(rctx):
+ """Query the xcode sdk location to update cflags
+
+ Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so.
+ Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
+ otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
+ """
+
+ # Only run on MacOS hosts
+ if not rctx.os.name.lower().startswith("mac os"):
+ return []
+
+ xcode_sdk_location = repo_utils.execute_unchecked(
+ rctx,
+ op = "GetXcodeLocation",
+ arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"],
+ )
+ if xcode_sdk_location.return_code != 0:
+ return []
+
+ xcode_root = xcode_sdk_location.stdout.strip()
+ if _COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower():
+ # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer
+ # so we need to change the path to to the macos specific tools which are in a different relative
+ # path than xcode installed command line tools.
+ xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root)
+ return [
+ "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root),
+ ]
+
+def _get_toolchain_unix_cflags(rctx, python_interpreter):
+ """Gather cflags from a standalone toolchain for unix systems.
+
+ Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
+ otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
+ """
+
+ # Only run on Unix systems
+ if not rctx.os.name.lower().startswith(("mac os", "linux")):
+ return []
+
+ # Only update the location when using a standalone toolchain.
+ if not is_standalone_interpreter(rctx, python_interpreter):
+ return []
+
+ stdout = repo_utils.execute_checked_stdout(
+ rctx,
+ op = "GetPythonVersionForUnixCflags",
+ arguments = [
+ python_interpreter,
+ "-c",
+ "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')",
+ ],
+ )
+ _python_version = stdout
+ include_path = "{}/include/python{}".format(
+ python_interpreter.dirname,
+ _python_version,
+ )
+
+ return ["-isystem {}".format(include_path)]
+
+def _parse_optional_attrs(rctx, args, extra_pip_args = None):
+ """Helper function to parse common attributes of pip_repository and whl_library repository rules.
+
+ This function also serializes the structured arguments as JSON
+ so they can be passed on the command line to subprocesses.
+
+ Args:
+ rctx: Handle to the rule repository context.
+ args: A list of parsed args for the rule.
+ extra_pip_args: The pip args to pass.
+ Returns: Augmented args list.
+ """
+
+ if use_isolated(rctx, rctx.attr):
+ args.append("--isolated")
+
+ # Bazel version 7.1.0 and later (and rolling releases from version 8.0.0-pre.20240128.3)
+ # support rctx.getenv(name, default): When building incrementally, any change to the value of
+ # the variable named by name will cause this repository to be re-fetched.
+ if "getenv" in dir(rctx):
+ getenv = rctx.getenv
+ else:
+ getenv = rctx.os.environ.get
+
+ # Check for None so we use empty default types from our attrs.
+ # Some args want to be list, and some want to be dict.
+ if extra_pip_args != None:
+ args += [
+ "--extra_pip_args",
+ json.encode(struct(arg = [
+ envsubst(pip_arg, rctx.attr.envsubst, getenv)
+ for pip_arg in rctx.attr.extra_pip_args
+ ])),
+ ]
+
+ if rctx.attr.download_only:
+ args.append("--download_only")
+
+ if rctx.attr.pip_data_exclude != None:
+ args += [
+ "--pip_data_exclude",
+ json.encode(struct(arg = rctx.attr.pip_data_exclude)),
+ ]
+
+ if rctx.attr.enable_implicit_namespace_pkgs:
+ args.append("--enable_implicit_namespace_pkgs")
+
+ if rctx.attr.environment != None:
+ args += [
+ "--environment",
+ json.encode(struct(arg = rctx.attr.environment)),
+ ]
+
+ return args
+
+def _create_repository_execution_environment(rctx, python_interpreter):
+ """Create a environment dictionary for processes we spawn with rctx.execute.
+
+ Args:
+ rctx (repository_ctx): The repository context.
+ python_interpreter (path): The resolved python interpreter.
+ Returns:
+ Dictionary of environment variable suitable to pass to rctx.execute.
+ """
+
+ # Gather any available CPPFLAGS values
+ cppflags = []
+ cppflags.extend(_get_xcode_location_cflags(rctx))
+ cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter))
+
+ env = {
+ "PYTHONPATH": _construct_pypath(rctx),
+ _CPPFLAGS: " ".join(cppflags),
+ }
+
+ return env
+
+def _whl_library_impl(rctx):
+ python_interpreter = _resolve_python_interpreter(rctx)
+ args = [
+ python_interpreter,
+ "-m",
+ "python.pip_install.tools.wheel_installer.wheel_installer",
+ "--requirement",
+ rctx.attr.requirement,
+ ]
+ extra_pip_args = []
+ extra_pip_args.extend(rctx.attr.extra_pip_args)
+
+ # Manually construct the PYTHONPATH since we cannot use the toolchain here
+ environment = _create_repository_execution_environment(rctx, python_interpreter)
+
+ whl_path = None
+ if rctx.attr.whl_file:
+ whl_path = rctx.path(rctx.attr.whl_file)
+
+ # Simulate the behaviour where the whl is present in the current directory.
+ rctx.symlink(whl_path, whl_path.basename)
+ whl_path = rctx.path(whl_path.basename)
+ elif rctx.attr.urls:
+ filename = rctx.attr.filename
+ urls = rctx.attr.urls
+ if not filename:
+ _, _, filename = urls[0].rpartition("/")
+
+ if not (filename.endswith(".whl") or filename.endswith("tar.gz") or filename.endswith(".zip")):
+ if rctx.attr.filename:
+ msg = "got '{}'".format(filename)
+ else:
+ msg = "detected '{}' from url:\n{}".format(filename, urls[0])
+ fail("Only '.whl', '.tar.gz' or '.zip' files are supported, {}".format(msg))
+
+ result = rctx.download(
+ url = urls,
+ output = filename,
+ sha256 = rctx.attr.sha256,
+ auth = get_auth(rctx, urls),
+ )
+
+ if not result.success:
+ fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))
+
+ if filename.endswith(".whl"):
+ whl_path = rctx.path(rctx.attr.filename)
+ else:
+ # It is an sdist and we need to tell PyPI to use a file in this directory
+ # and not use any indexes.
+ extra_pip_args.extend(["--no-index", "--find-links", "."])
+
+ args = _parse_optional_attrs(rctx, args, extra_pip_args)
+
+ if not whl_path:
+ repo_utils.execute_checked(
+ rctx,
+ op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement),
+ arguments = args,
+ environment = environment,
+ quiet = rctx.attr.quiet,
+ timeout = rctx.attr.timeout,
+ )
+
+ whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
+ if not rctx.delete("whl_file.json"):
+ fail("failed to delete the whl_file.json file")
+
+ if rctx.attr.whl_patches:
+ patches = {}
+ for patch_file, json_args in rctx.attr.whl_patches.items():
+ patch_dst = struct(**json.decode(json_args))
+ if whl_path.basename in patch_dst.whls:
+ patches[patch_file] = patch_dst.patch_strip
+
+ whl_path = patch_whl(
+ rctx,
+ op = "whl_library.PatchWhl({}, {})".format(rctx.attr.name, rctx.attr.requirement),
+ python_interpreter = python_interpreter,
+ whl_path = whl_path,
+ patches = patches,
+ quiet = rctx.attr.quiet,
+ timeout = rctx.attr.timeout,
+ )
+
+ target_platforms = rctx.attr.experimental_target_platforms
+ if target_platforms:
+ parsed_whl = parse_whl_name(whl_path.basename)
+ if parsed_whl.platform_tag != "any":
+ # NOTE @aignas 2023-12-04: if the wheel is a platform specific
+ # wheel, we only include deps for that target platform
+ target_platforms = [
+ p.target_platform
+ for p in whl_target_platforms(
+ platform_tag = parsed_whl.platform_tag,
+ abi_tag = parsed_whl.abi_tag,
+ )
+ ]
+
+ repo_utils.execute_checked(
+ rctx,
+ op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
+ arguments = args + [
+ "--whl-file",
+ whl_path,
+ ] + ["--platform={}".format(p) for p in target_platforms],
+ environment = environment,
+ quiet = rctx.attr.quiet,
+ timeout = rctx.attr.timeout,
+ )
+
+ metadata = json.decode(rctx.read("metadata.json"))
+ rctx.delete("metadata.json")
+
+ build_file_contents = generate_whl_library_build_bazel(
+ dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
+ whl_name = whl_path.basename,
+ dependencies = metadata["deps"],
+ dependencies_by_platform = metadata["deps_by_platform"],
+ group_name = rctx.attr.group_name,
+ group_deps = rctx.attr.group_deps,
+ data_exclude = rctx.attr.pip_data_exclude,
+ tags = [
+ "pypi_name=" + metadata["name"],
+ "pypi_version=" + metadata["version"],
+ ],
+ entry_points = {},
+ annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
+ )
+ rctx.file("BUILD.bazel", build_file_contents)
+
+ return
+
+# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique
+whl_library_attrs = dict({
+ "annotation": attr.label(
+ doc = (
+ "Optional json encoded file containing annotation to apply to the extracted wheel. " +
+ "See `package_annotation`"
+ ),
+ allow_files = True,
+ ),
+ "dep_template": attr.string(
+ doc = """
+The dep template to use for referencing the dependencies. It should have `{name}`
+and `{target}` tokens that will be replaced with the normalized distribution name
+and the target that we need respectively.
+""",
+ ),
+ "filename": attr.string(
+ doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.",
+ ),
+ "group_deps": attr.string_list(
+ doc = "List of dependencies to skip in order to break the cycles within a dependency group.",
+ default = [],
+ ),
+ "group_name": attr.string(
+ doc = "Name of the group, if any.",
+ ),
+ "repo": attr.string(
+ mandatory = True,
+ doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.",
+ ),
+ "repo_prefix": attr.string(
+ doc = """
+Prefix for the generated packages will be of the form `@//...`
+
+DEPRECATED. Only left for people who vendor requirements.bzl.
+""",
+ ),
+ "requirement": attr.string(
+ mandatory = True,
+ doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.",
+ ),
+ "sha256": attr.string(
+ doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.",
+ ),
+ "urls": attr.string_list(
+ doc = """\
+The list of urls of the whl to be downloaded using bazel downloader. Using this
+attr makes `extra_pip_args` and `download_only` ignored.""",
+ ),
+ "whl_file": attr.label(
+ doc = "The whl file that should be used instead of downloading or building the whl.",
+ ),
+ "whl_patches": attr.label_keyed_string_dict(
+ doc = """a label-keyed-string dict that has
+ json.encode(struct([whl_file], patch_strip]) as values. This
+ is to maintain flexibility and correct bzlmod extension interface
+ until we have a better way to define whl_library and move whl
+ patching to a separate place. INTERNAL USE ONLY.""",
+ ),
+ "_python_path_entries": attr.label_list(
+ # Get the root directory of these rules and keep them as a default attribute
+ # in order to avoid unnecessary repository fetching restarts.
+ #
+ # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478
+ default = [
+ Label("//:BUILD.bazel"),
+ ] + [
+ # Includes all the external dependencies from repositories.bzl
+ Label("@" + repo + "//:BUILD.bazel")
+ for repo in all_requirements
+ ],
+ ),
+}, **ATTRS)
+whl_library_attrs.update(AUTH_ATTRS)
+
+whl_library = repository_rule(
+ attrs = whl_library_attrs,
+ doc = """
+Download and extracts a single wheel based into a bazel repo based on the requirement string passed in.
+Instantiated from pip_repository and inherits config options from there.""",
+ implementation = _whl_library_impl,
+ environ = [
+ "RULES_PYTHON_PIP_ISOLATED",
+ REPO_DEBUG_ENV_VAR,
+ ],
+)
diff --git a/python/private/pypi/whl_library_utils.bzl b/python/private/pypi/whl_library_utils.bzl
deleted file mode 100644
index 7920224b95..0000000000
--- a/python/private/pypi/whl_library_utils.bzl
+++ /dev/null
@@ -1,18 +0,0 @@
-"""A generated file containing all source files used for `@rules_python//python/private/pypi:whl_library.bzl` rules
-
-This file is auto-generated from the `@rules_python//tests/pypi/whl_library_utils:srcs.update` target. Please
-`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
-"""
-
-# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
-# sources changed.
-PY_SRCS = [
- "@rules_python//python/private/pypi:repack_whl.py",
- "@rules_python//python/pip_install/tools/dependency_resolver:__init__.py",
- "@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py",
- "@rules_python//python/pip_install/tools/wheel_installer:arguments.py",
- "@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py",
- "@rules_python//python/pip_install/tools/wheel_installer:wheel.py",
- "@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py",
- "@rules_python//tools:wheelmaker.py",
-]
diff --git a/python/private/pypi/whl_library_utils_goal.bzl b/python/private/pypi/whl_library_utils_goal.bzl
deleted file mode 100644
index db1cc4addd..0000000000
--- a/python/private/pypi/whl_library_utils_goal.bzl
+++ /dev/null
@@ -1,132 +0,0 @@
-# Copyright 2023 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for `rules_python` whl_library repository rule"""
-
-load("//python/private:text_util.bzl", "render")
-
-_SRCS_TEMPLATE = """\
-\"\"\"A generated file containing all source files used for `@rules_python//python/private/pypi:whl_library.bzl` rules
-
-This file is auto-generated from the `@rules_python//tests/pypi/whl_library_utils:srcs.update` target. Please
-`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
-"\"\"
-
-# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
-# sources changed.
-PY_SRCS = {srcs}
-"""
-
-def _src_label(file):
- dir_path, file_name = file.short_path.rsplit("/", 1)
-
- return "@rules_python//{}:{}".format(
- dir_path,
- file_name,
- )
-
-def _srcs_module_impl(ctx):
- srcs = [_src_label(src) for src in ctx.files.srcs]
- if not srcs:
- fail("`srcs` cannot be empty")
- output = ctx.actions.declare_file(ctx.label.name)
-
- ctx.actions.write(
- output = output,
- content = _SRCS_TEMPLATE.format(
- srcs = render.list(srcs),
- ),
- )
-
- return DefaultInfo(
- files = depset([output]),
- )
-
-_srcs_module = rule(
- doc = "A rule for writing a list of sources to a templated file",
- implementation = _srcs_module_impl,
- attrs = {
- "srcs": attr.label(
- doc = "A filegroup of source files",
- allow_files = True,
- ),
- },
-)
-
-_INSTALLER_TEMPLATE = """\
-#!/bin/bash
-set -euo pipefail
-cp -f "{path}" "${{BUILD_WORKSPACE_DIRECTORY}}/{dest}"
-"""
-
-def _srcs_updater_impl(ctx):
- output = ctx.actions.declare_file(ctx.label.name + ".sh")
- target_file = ctx.file.input
- dest = ctx.file.dest.short_path
-
- ctx.actions.write(
- output = output,
- content = _INSTALLER_TEMPLATE.format(
- path = target_file.short_path,
- dest = dest,
- ),
- is_executable = True,
- )
-
- return DefaultInfo(
- files = depset([output]),
- runfiles = ctx.runfiles(files = [target_file]),
- executable = output,
- )
-
-_srcs_updater = rule(
- doc = "A rule for writing a `whl_library_utils.bzl` file back to the repository",
- implementation = _srcs_updater_impl,
- attrs = {
- "dest": attr.label(
- doc = "The target file to write the new `input` to.",
- allow_single_file = ["whl_library_utils.bzl"],
- mandatory = True,
- ),
- "input": attr.label(
- doc = "The file to write back to the repository",
- allow_single_file = True,
- mandatory = True,
- ),
- },
- executable = True,
-)
-
-def whl_library_utils_goal(name, dest, **kwargs):
- """A helper rule to ensure `pip_repository` rules are always up to date
-
- Args:
- name (str): The name of the sources module
- dest (str): The filename the module should be written as in the current package.
- **kwargs (dict): Additional keyword arguments
- """
- tags = kwargs.pop("tags", [])
-
- _srcs_module(
- name = name,
- tags = tags,
- **kwargs
- )
-
- _srcs_updater(
- name = name + ".update",
- input = name,
- dest = dest,
- tags = tags,
- )
diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl
index dade9cba9b..702a08e281 100644
--- a/python/private/text_util.bzl
+++ b/python/private/text_util.bzl
@@ -36,6 +36,9 @@ def _render_alias(name, actual, *, visibility = None):
])
def _render_dict(d, *, key_repr = repr, value_repr = repr):
+ if not d:
+ return "{}"
+
return "\n".join([
"{",
_indent("\n".join([
diff --git a/tests/pypi/whl_library_utils/BUILD.bazel b/tests/pypi/whl_library_utils/BUILD.bazel
deleted file mode 100644
index 155e8afacf..0000000000
--- a/tests/pypi/whl_library_utils/BUILD.bazel
+++ /dev/null
@@ -1,17 +0,0 @@
-load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
-
-diff_test(
- name = "whl_library_utils_test",
- failure_message = (
- "Please run 'bazel run //python/private/pypi:whl_library_utils.update' " +
- "to update the 'whl_library_utils.bzl' module found in the same package."
- ),
- file1 = "//python/private/pypi:whl_library_utils",
- file2 = "//python/private/pypi:whl_library_utils.bzl",
- # TODO: The diff_test here fails on Windows. As does the
- # install script. This should be fixed.
- target_compatible_with = select({
- "@platforms//os:windows": ["@platforms//:incompatible"],
- "//conditions:default": [],
- }),
-)
From 9032fc275d98bf8cbbc07206357ac046d155fb06 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 17:18:18 +0900
Subject: [PATCH 19/25] move the bzlmod extension to private/pypi
---
python/extensions/BUILD.bazel | 2 -
python/private/bzlmod/BUILD.bazel | 33 +-
python/private/bzlmod/pip.bzl | 806 +----------------
python/private/pypi/BUILD.bazel | 34 +-
python/private/pypi/bzlmod.bzl | 818 ++++++++++++++++++
.../hub_repository.bzl} | 50 +-
.../requirements.bzl.tmpl.bzlmod} | 0
7 files changed, 882 insertions(+), 861 deletions(-)
create mode 100644 python/private/pypi/bzlmod.bzl
rename python/private/{bzlmod/pip_repository.bzl => pypi/hub_repository.bzl} (79%)
rename python/private/{bzlmod/requirements.bzl.tmpl => pypi/requirements.bzl.tmpl.bzlmod} (100%)
diff --git a/python/extensions/BUILD.bazel b/python/extensions/BUILD.bazel
index eb095ab746..1bc2a71192 100644
--- a/python/extensions/BUILD.bazel
+++ b/python/extensions/BUILD.bazel
@@ -36,8 +36,6 @@ bzl_library(
srcs = ["python.bzl"],
visibility = ["//:__subpackages__"],
deps = [
- "//python/private:util_bzl",
- "//python/private/bzlmod:bazel_features_bzl",
"//python/private/bzlmod:python_bzl",
],
)
diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel
index b76e905998..2cb35fc03e 100644
--- a/python/private/bzlmod/BUILD.bazel
+++ b/python/private/bzlmod/BUILD.bazel
@@ -13,7 +13,6 @@
# limitations under the License.
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
package(default_visibility = ["//:__subpackages__"])
@@ -29,35 +28,7 @@ bzl_library(
name = "pip_bzl",
srcs = ["pip.bzl"],
deps = [
- ":pip_repository_bzl",
- "//python/private/pypi:pip_repository_attrs_bzl",
- "//python/private/pypi:whl_library_bzl",
- "//python/private/pypi:attrs_bzl",
- "//python/private/pypi:simpleapi_download_bzl",
- "//python/private:full_version_bzl",
- "//python/private:normalize_name_bzl",
- "//python/private/pypi:parse_requirements_bzl",
- "//python/private/pypi:parse_whl_name_bzl",
- "//python/private/pypi:whl_repo_name_bzl",
- "//python/private:version_label_bzl",
- ":bazel_features_bzl",
- ] + [
- "@pythons_hub//:interpreters_bzl",
- ] if BZLMOD_ENABLED else [],
-)
-
-bzl_library(
- name = "bazel_features_bzl",
- deps = ["@bazel_features//:features"],
-)
-
-bzl_library(
- name = "pip_repository_bzl",
- srcs = ["pip_repository.bzl"],
- visibility = ["//:__subpackages__"],
- deps = [
- "//python/private:text_util_bzl",
- "//python/private/pypi:render_pkg_aliases_bzl",
+ "//python/private/pypi:bzlmod_bzl",
],
)
@@ -68,6 +39,8 @@ bzl_library(
":pythons_hub_bzl",
"//python:repositories_bzl",
"//python/private:toolchains_repo_bzl",
+ "//python/private:util_bzl",
+ "@bazel_features//:features",
],
)
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index 187a48e7a2..ecf94b69d5 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -12,807 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"pip module extension for use with bzlmod"
+"pip module extensions for use with bzlmod."
-load("@bazel_features//:features.bzl", "bazel_features")
-load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS")
-load("//python/private:auth.bzl", "AUTH_ATTRS")
-load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:repo_utils.bzl", "repo_utils")
-load("//python/private:version_label.bzl", "version_label")
-load("//python/private/pypi:attrs.bzl", "use_isolated")
-load("//python/private/pypi:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
-load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name")
-load("//python/private/pypi:pip_repository_attrs.bzl", "ATTRS")
-load("//python/private/pypi:render_pkg_aliases.bzl", "whl_alias")
-load("//python/private/pypi:simpleapi_download.bzl", "simpleapi_download")
-load("//python/private/pypi:whl_library.bzl", "whl_library")
-load("//python/private/pypi:whl_repo_name.bzl", "whl_repo_name")
-load(":pip_repository.bzl", "pip_repository")
+load("//python/private/pypi:bzlmod.bzl", "pypi", "pypi_internal")
-def _parse_version(version):
- major, _, version = version.partition(".")
- minor, _, version = version.partition(".")
- patch, _, version = version.partition(".")
- build, _, version = version.partition(".")
-
- return struct(
- # use semver vocabulary here
- major = major,
- minor = minor,
- patch = patch, # this is called `micro` in the Python interpreter versioning scheme
- build = build,
- )
-
-def _major_minor_version(version):
- version = _parse_version(version)
- return "{}.{}".format(version.major, version.minor)
-
-def _whl_mods_impl(mctx):
- """Implementation of the pip.whl_mods tag class.
-
- This creates the JSON files used to modify the creation of different wheels.
-"""
- whl_mods_dict = {}
- for mod in mctx.modules:
- for whl_mod_attr in mod.tags.whl_mods:
- if whl_mod_attr.hub_name not in whl_mods_dict.keys():
- whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr}
- elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys():
- # We cannot have the same wheel name in the same hub, as we
- # will create the same JSON file name.
- fail("""\
-Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format(
- whl_mod_attr.whl_name,
- whl_mod_attr.hub_name,
- ))
- else:
- whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr
-
- for hub_name, whl_maps in whl_mods_dict.items():
- whl_mods = {}
-
- # create a struct that we can pass to the _whl_mods_repo rule
- # to create the different JSON files.
- for whl_name, mods in whl_maps.items():
- build_content = mods.additive_build_content
- if mods.additive_build_content_file != None and mods.additive_build_content != "":
- fail("""\
-You cannot use both the additive_build_content and additive_build_content_file arguments at the same time.
-""")
- elif mods.additive_build_content_file != None:
- build_content = mctx.read(mods.additive_build_content_file)
-
- whl_mods[whl_name] = json.encode(struct(
- additive_build_content = build_content,
- copy_files = mods.copy_files,
- copy_executables = mods.copy_executables,
- data = mods.data,
- data_exclude_glob = mods.data_exclude_glob,
- srcs_exclude_glob = mods.srcs_exclude_glob,
- ))
-
- _whl_mods_repo(
- name = hub_name,
- whl_mods = whl_mods,
- )
-
-def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache):
- logger = repo_utils.logger(module_ctx)
- python_interpreter_target = pip_attr.python_interpreter_target
- is_hub_reproducible = True
-
- # if we do not have the python_interpreter set in the attributes
- # we programmatically find it.
- hub_name = pip_attr.hub_name
- if python_interpreter_target == None and not pip_attr.python_interpreter:
- python_name = "python_{}_host".format(
- pip_attr.python_version.replace(".", "_"),
- )
- if python_name not in INTERPRETER_LABELS:
- fail((
- "Unable to find interpreter for pip hub '{hub_name}' for " +
- "python_version={version}: Make sure a corresponding " +
- '`python.toolchain(python_version="{version}")` call exists.' +
- "Expected to find {python_name} among registered versions:\n {labels}"
- ).format(
- hub_name = hub_name,
- version = pip_attr.python_version,
- python_name = python_name,
- labels = " \n".join(INTERPRETER_LABELS),
- ))
- python_interpreter_target = INTERPRETER_LABELS[python_name]
-
- pip_name = "{}_{}".format(
- hub_name,
- version_label(pip_attr.python_version),
- )
- major_minor = _major_minor_version(pip_attr.python_version)
-
- if hub_name not in whl_map:
- whl_map[hub_name] = {}
-
- whl_modifications = {}
- if pip_attr.whl_modifications != None:
- for mod, whl_name in pip_attr.whl_modifications.items():
- whl_modifications[whl_name] = mod
-
- if pip_attr.experimental_requirement_cycles:
- requirement_cycles = {
- name: [normalize_name(whl_name) for whl_name in whls]
- for name, whls in pip_attr.experimental_requirement_cycles.items()
- }
-
- whl_group_mapping = {
- whl_name: group_name
- for group_name, group_whls in requirement_cycles.items()
- for whl_name in group_whls
- }
-
- # TODO @aignas 2024-04-05: how do we support different requirement
- # cycles for different abis/oses? For now we will need the users to
- # assume the same groups across all versions/platforms until we start
- # using an alternative cycle resolution strategy.
- group_map[hub_name] = pip_attr.experimental_requirement_cycles
- else:
- whl_group_mapping = {}
- requirement_cycles = {}
-
- # Create a new wheel library for each of the different whls
-
- get_index_urls = None
- if pip_attr.experimental_index_url:
- if pip_attr.download_only:
- fail("Currently unsupported to use `download_only` and `experimental_index_url`")
-
- get_index_urls = lambda ctx, distributions: simpleapi_download(
- ctx,
- attr = struct(
- index_url = pip_attr.experimental_index_url,
- extra_index_urls = pip_attr.experimental_extra_index_urls or [],
- index_url_overrides = pip_attr.experimental_index_url_overrides or {},
- sources = distributions,
- envsubst = pip_attr.envsubst,
- # Auth related info
- netrc = pip_attr.netrc,
- auth_patterns = pip_attr.auth_patterns,
- ),
- cache = simpleapi_cache,
- parallel_download = pip_attr.parallel_download,
- )
-
- requirements_by_platform = parse_requirements(
- module_ctx,
- requirements_by_platform = pip_attr.requirements_by_platform,
- requirements_linux = pip_attr.requirements_linux,
- requirements_lock = pip_attr.requirements_lock,
- requirements_osx = pip_attr.requirements_darwin,
- requirements_windows = pip_attr.requirements_windows,
- extra_pip_args = pip_attr.extra_pip_args,
- get_index_urls = get_index_urls,
- python_version = major_minor,
- logger = logger,
- )
-
- repository_platform = host_platform(module_ctx.os)
- for whl_name, requirements in requirements_by_platform.items():
- # We are not using the "sanitized name" because the user
- # would need to guess what name we modified the whl name
- # to.
- annotation = whl_modifications.get(whl_name)
- whl_name = normalize_name(whl_name)
-
- group_name = whl_group_mapping.get(whl_name)
- group_deps = requirement_cycles.get(group_name, [])
-
- # Construct args separately so that the lock file can be smaller and does not include unused
- # attrs.
- whl_library_args = dict(
- repo = pip_name,
- dep_template = "@{}//{{name}}:{{target}}".format(hub_name),
- )
- maybe_args = dict(
- # The following values are safe to omit if they have false like values
- annotation = annotation,
- download_only = pip_attr.download_only,
- enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
- environment = pip_attr.environment,
- envsubst = pip_attr.envsubst,
- experimental_target_platforms = pip_attr.experimental_target_platforms,
- group_deps = group_deps,
- group_name = group_name,
- pip_data_exclude = pip_attr.pip_data_exclude,
- python_interpreter = pip_attr.python_interpreter,
- python_interpreter_target = python_interpreter_target,
- whl_patches = {
- p: json.encode(args)
- for p, args in whl_overrides.get(whl_name, {}).items()
- },
- )
- whl_library_args.update({k: v for k, v in maybe_args.items() if v})
- maybe_args_with_default = dict(
- # The following values have defaults next to them
- isolated = (use_isolated(module_ctx, pip_attr), True),
- quiet = (pip_attr.quiet, True),
- timeout = (pip_attr.timeout, 600),
- )
- whl_library_args.update({
- k: v
- for k, (v, default) in maybe_args_with_default.items()
- if v != default
- })
-
- if get_index_urls:
- # TODO @aignas 2024-05-26: move to a separate function
- found_something = False
- for requirement in requirements:
- for distribution in requirement.whls + [requirement.sdist]:
- if not distribution:
- # sdist may be None
- continue
-
- found_something = True
- is_hub_reproducible = False
-
- if pip_attr.netrc:
- whl_library_args["netrc"] = pip_attr.netrc
- if pip_attr.auth_patterns:
- whl_library_args["auth_patterns"] = pip_attr.auth_patterns
-
- # pip is not used to download wheels and the python `whl_library` helpers are only extracting things
- whl_library_args.pop("extra_pip_args", None)
-
- # This is no-op because pip is not used to download the wheel.
- whl_library_args.pop("download_only", None)
-
- repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256)
- whl_library_args["requirement"] = requirement.srcs.requirement
- whl_library_args["urls"] = [distribution.url]
- whl_library_args["sha256"] = distribution.sha256
- whl_library_args["filename"] = distribution.filename
- whl_library_args["experimental_target_platforms"] = requirement.target_platforms
-
- # Pure python wheels or sdists may need to have a platform here
- target_platforms = None
- if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
- if len(requirements) > 1:
- target_platforms = requirement.target_platforms
-
- whl_library(name = repo_name, **dict(sorted(whl_library_args.items())))
-
- whl_map[hub_name].setdefault(whl_name, []).append(
- whl_alias(
- repo = repo_name,
- version = major_minor,
- filename = distribution.filename,
- target_platforms = target_platforms,
- ),
- )
-
- if found_something:
- continue
-
- requirement = select_requirement(
- requirements,
- platform = repository_platform,
- )
- if not requirement:
- # Sometimes the package is not present for host platform if there
- # are whls specified only in particular requirements files, in that
- # case just continue, however, if the download_only flag is set up,
- # then the user can also specify the target platform of the wheel
- # packages they want to download, in that case there will be always
- # a requirement here, so we will not be in this code branch.
- continue
- elif get_index_urls:
- logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
-
- whl_library_args["requirement"] = requirement.requirement_line
- if requirement.extra_pip_args:
- whl_library_args["extra_pip_args"] = requirement.extra_pip_args
-
- # We sort so that the lock-file remains the same no matter the order of how the
- # args are manipulated in the code going before.
- repo_name = "{}_{}".format(pip_name, whl_name)
- whl_library(name = repo_name, **dict(sorted(whl_library_args.items())))
- whl_map[hub_name].setdefault(whl_name, []).append(
- whl_alias(
- repo = repo_name,
- version = major_minor,
- ),
- )
-
- return is_hub_reproducible
-
-def _pip_impl(module_ctx):
- """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
-
- This implementation iterates through all of the `pip.parse` calls and creates
- different pip hub repositories based on the "hub_name". Each of the
- pip calls create spoke repos that uses a specific Python interpreter.
-
- In a MODULES.bazel file we have:
-
- pip.parse(
- hub_name = "pip",
- python_version = 3.9,
- requirements_lock = "//:requirements_lock_3_9.txt",
- requirements_windows = "//:requirements_windows_3_9.txt",
- )
- pip.parse(
- hub_name = "pip",
- python_version = 3.10,
- requirements_lock = "//:requirements_lock_3_10.txt",
- requirements_windows = "//:requirements_windows_3_10.txt",
- )
-
- For instance, we have a hub with the name of "pip".
- A repository named the following is created. It is actually called last when
- all of the pip spokes are collected.
-
- - @@rules_python~override~pip~pip
-
- As shown in the example code above we have the following.
- Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip".
- These definitions create two different pip spoke repositories that are
- related to the hub "pip".
- One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically
- determines the Python version and the interpreter.
- Both of these pip spokes contain requirements files that includes websocket
- and its dependencies.
-
- We also need repositories for the wheels that the different pip spokes contain.
- For each Python version a different wheel repository is created. In our example
- each pip spoke had a requirements file that contained websockets. We
- then create two different wheel repositories that are named the following.
-
- - @@rules_python~override~pip~pip_39_websockets
- - @@rules_python~override~pip~pip_310_websockets
-
- And if the wheel has any other dependencies subsequent wheels are created in the same fashion.
-
- The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to
- a spoke repository depending on the Python version.
-
- Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple
- hubs pointing to various different pip spokes.
-
- Some other business rules notes. A hub can only have one spoke per Python version. We cannot
- have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second
- we cannot have the same hub name used in sub-modules. The hub name has to be globally
- unique.
-
- This implementation also handles the creation of whl_modification JSON files that are used
- during the creation of wheel libraries. These JSON files used via the annotations argument
- when calling wheel_installer.py.
-
- Args:
- module_ctx: module contents
- """
-
- # Build all of the wheel modifications if the tag class is called.
- _whl_mods_impl(module_ctx)
-
- _overriden_whl_set = {}
- whl_overrides = {}
-
- for module in module_ctx.modules:
- for attr in module.tags.override:
- if not module.is_root:
- fail("overrides are only supported in root modules")
-
- if not attr.file.endswith(".whl"):
- fail("Only whl overrides are supported at this time")
-
- whl_name = normalize_name(parse_whl_name(attr.file).distribution)
-
- if attr.file in _overriden_whl_set:
- fail("Duplicate module overrides for '{}'".format(attr.file))
- _overriden_whl_set[attr.file] = None
-
- for patch in attr.patches:
- if whl_name not in whl_overrides:
- whl_overrides[whl_name] = {}
-
- if patch not in whl_overrides[whl_name]:
- whl_overrides[whl_name][patch] = struct(
- patch_strip = attr.patch_strip,
- whls = [],
- )
-
- whl_overrides[whl_name][patch].whls.append(attr.file)
-
- # Used to track all the different pip hubs and the spoke pip Python
- # versions.
- pip_hub_map = {}
-
- # Keeps track of all the hub's whl repos across the different versions.
- # dict[hub, dict[whl, dict[version, str pip]]]
- # Where hub, whl, and pip are the repo names
- hub_whl_map = {}
- hub_group_map = {}
-
- simpleapi_cache = {}
- is_extension_reproducible = True
-
- for mod in module_ctx.modules:
- for pip_attr in mod.tags.parse:
- hub_name = pip_attr.hub_name
- if hub_name not in pip_hub_map:
- pip_hub_map[pip_attr.hub_name] = struct(
- module_name = mod.name,
- python_versions = [pip_attr.python_version],
- )
- elif pip_hub_map[hub_name].module_name != mod.name:
- # We cannot have two hubs with the same name in different
- # modules.
- fail((
- "Duplicate cross-module pip hub named '{hub}': pip hub " +
- "names must be unique across modules. First defined " +
- "by module '{first_module}', second attempted by " +
- "module '{second_module}'"
- ).format(
- hub = hub_name,
- first_module = pip_hub_map[hub_name].module_name,
- second_module = mod.name,
- ))
-
- elif pip_attr.python_version in pip_hub_map[hub_name].python_versions:
- fail((
- "Duplicate pip python version '{version}' for hub " +
- "'{hub}' in module '{module}': the Python versions " +
- "used for a hub must be unique"
- ).format(
- hub = hub_name,
- module = mod.name,
- version = pip_attr.python_version,
- ))
- else:
- pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
-
- is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
- is_extension_reproducible = is_extension_reproducible and is_hub_reproducible
-
- for hub_name, whl_map in hub_whl_map.items():
- pip_repository(
- name = hub_name,
- repo_name = hub_name,
- whl_map = {
- key: json.encode(value)
- for key, value in whl_map.items()
- },
- default_version = _major_minor_version(DEFAULT_PYTHON_VERSION),
- groups = hub_group_map.get(hub_name),
- )
-
- if bazel_features.external_deps.extension_metadata_has_reproducible:
- # If we are not using the `experimental_index_url feature, the extension is fully
- # deterministic and we don't need to create a lock entry for it.
- #
- # In order to be able to dogfood the `experimental_index_url` feature before it gets
- # stabilized, we have created the `_pip_non_reproducible` function, that will result
- # in extra entries in the lock file.
- return module_ctx.extension_metadata(reproducible = is_extension_reproducible)
- else:
- return None
-
-def _pip_non_reproducible(module_ctx):
- _pip_impl(module_ctx)
-
- # We default to calling the PyPI index and that will go into the
- # MODULE.bazel.lock file, hence return nothing here.
- return None
-
-def _pip_parse_ext_attrs(**kwargs):
- """Get the attributes for the pip extension.
-
- Args:
- **kwargs: A kwarg for setting defaults for the specific attributes. The
- key is expected to be the same as the attribute key.
-
- Returns:
- A dict of attributes.
- """
- attrs = dict({
- "experimental_extra_index_urls": attr.string_list(
- doc = """\
-The extra index URLs to use for downloading wheels using bazel downloader.
-Each value is going to be subject to `envsubst` substitutions if necessary.
-
-The indexes must support Simple API as described here:
-https://packaging.python.org/en/latest/specifications/simple-repository-api/
-
-This is equivalent to `--extra-index-urls` `pip` option.
-""",
- default = [],
- ),
- "experimental_index_url": attr.string(
- default = kwargs.get("experimental_index_url", ""),
- doc = """\
-The index URL to use for downloading wheels using bazel downloader. This value is going
-to be subject to `envsubst` substitutions if necessary.
-
-The indexes must support Simple API as described here:
-https://packaging.python.org/en/latest/specifications/simple-repository-api/
-
-In the future this could be defaulted to `https://pypi.org` when this feature becomes
-stable.
-
-This is equivalent to `--index-url` `pip` option.
-""",
- ),
- "experimental_index_url_overrides": attr.string_dict(
- doc = """\
-The index URL overrides for each package to use for downloading wheels using
-bazel downloader. This value is going to be subject to `envsubst` substitutions
-if necessary.
-
-The key is the package name (will be normalized before usage) and the value is the
-index URL.
-
-This design pattern has been chosen in order to be fully deterministic about which
-packages come from which source. We want to avoid issues similar to what happened in
-https://pytorch.org/blog/compromised-nightly-dependency/.
-
-The indexes must support Simple API as described here:
-https://packaging.python.org/en/latest/specifications/simple-repository-api/
-""",
- ),
- "hub_name": attr.string(
- mandatory = True,
- doc = """
-The name of the repo pip dependencies will be accessible from.
-
-This name must be unique between modules; unless your module is guaranteed to
-always be the root module, it's highly recommended to include your module name
-in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can
-be used for shorter local names within your module.
-
-Within a module, the same `hub_name` can be specified to group different Python
-versions of pip dependencies under one repository name. This allows using a
-Python version-agnostic name when referring to pip dependencies; the
-correct version will be automatically selected.
-
-Typically, a module will only have a single hub of pip dependencies, but this
-is not required. Each hub is a separate resolution of pip dependencies. This
-means if different programs need different versions of some library, separate
-hubs can be created, and each program can use its respective hub's targets.
-Targets from different hubs should not be used together.
-""",
- ),
- "parallel_download": attr.bool(
- doc = """\
-The flag allows to make use of parallel downloading feature in bazel 7.1 and above
-when the bazel downloader is used. This is by default enabled as it improves the
-performance by a lot, but in case the queries to the simple API are very expensive
-or when debugging authentication issues one may want to disable this feature.
-
-NOTE, This will download (potentially duplicate) data for multiple packages if
-there is more than one index available, but in general this should be negligible
-because the simple API calls are very cheap and the user should not notice any
-extra overhead.
-
-If we are in synchronous mode, then we will use the first result that we
-find in case extra indexes are specified.
-""",
- default = True,
- ),
- "python_version": attr.string(
- mandatory = True,
- doc = """
-The Python version the dependencies are targetting, in Major.Minor format
-(e.g., "3.11") or patch level granularity (e.g. "3.11.1").
-
-If an interpreter isn't explicitly provided (using `python_interpreter` or
-`python_interpreter_target`), then the version specified here must have
-a corresponding `python.toolchain()` configured.
-""",
- ),
- "whl_modifications": attr.label_keyed_string_dict(
- mandatory = False,
- doc = """\
-A dict of labels to wheel names that is typically generated by the whl_modifications.
-The labels are JSON config files describing the modifications.
-""",
- ),
- }, **ATTRS)
- attrs.update(AUTH_ATTRS)
-
- return attrs
-
-def _whl_mod_attrs():
- attrs = {
- "additive_build_content": attr.string(
- doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.",
- ),
- "additive_build_content_file": attr.label(
- doc = """\
-(label, optional): path to a BUILD file to add to the generated
-`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file
-arguments at the same time.""",
- ),
- "copy_executables": attr.string_dict(
- doc = """\
-(dict, optional): A mapping of `src` and `out` files for
-[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
-executable.""",
- ),
- "copy_files": attr.string_dict(
- doc = """\
-(dict, optional): A mapping of `src` and `out` files for
-[@bazel_skylib//rules:copy_file.bzl][cf]""",
- ),
- "data": attr.string_list(
- doc = """\
-(list, optional): A list of labels to add as `data` dependencies to
-the generated `py_library` target.""",
- ),
- "data_exclude_glob": attr.string_list(
- doc = """\
-(list, optional): A list of exclude glob patterns to add as `data` to
-the generated `py_library` target.""",
- ),
- "hub_name": attr.string(
- doc = """\
-Name of the whl modification, hub we use this name to set the modifications for
-pip.parse. If you have different pip hubs you can use a different name,
-otherwise it is best practice to just use one.
-
-You cannot have the same `hub_name` in different modules. You can reuse the same
-name in the same module for different wheels that you put in the same hub, but you
-cannot have a child module that uses the same `hub_name`.
-""",
- mandatory = True,
- ),
- "srcs_exclude_glob": attr.string_list(
- doc = """\
-(list, optional): A list of labels to add as `srcs` to the generated
-`py_library` target.""",
- ),
- "whl_name": attr.string(
- doc = "The whl name that the modifications are used for.",
- mandatory = True,
- ),
- }
- return attrs
-
-# NOTE: the naming of 'override' is taken from the bzlmod native
-# 'archive_override', 'git_override' bzlmod functions.
-_override_tag = tag_class(
- attrs = {
- "file": attr.string(
- doc = """\
-The Python distribution file name which needs to be patched. This will be
-applied to all repositories that setup this distribution via the pip.parse tag
-class.""",
- mandatory = True,
- ),
- "patch_strip": attr.int(
- default = 0,
- doc = """\
-The number of leading path segments to be stripped from the file name in the
-patches.""",
- ),
- "patches": attr.label_list(
- doc = """\
-A list of patches to apply to the repository *after* 'whl_library' is extracted
-and BUILD.bazel file is generated.""",
- mandatory = True,
- ),
- },
- doc = """\
-Apply any overrides (e.g. patches) to a given Python distribution defined by
-other tags in this extension.""",
-)
-
-pip = module_extension(
- doc = """\
-This extension is used to make dependencies from pip available.
-
-pip.parse:
-To use, call `pip.parse()` and specify `hub_name` and your requirements file.
-Dependencies will be downloaded and made available in a repo named after the
-`hub_name` argument.
-
-Each `pip.parse()` call configures a particular Python version. Multiple calls
-can be made to configure different Python versions, and will be grouped by
-the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy`
-to automatically resolve to different, Python version-specific, libraries.
-
-pip.whl_mods:
-This tag class is used to help create JSON files to describe modifications to
-the BUILD files for wheels.
-""",
- implementation = _pip_impl,
- tag_classes = {
- "override": _override_tag,
- "parse": tag_class(
- attrs = _pip_parse_ext_attrs(),
- doc = """\
-This tag class is used to create a pip hub and all of the spokes that are part of that hub.
-This tag class reuses most of the pip attributes that are found in
-@rules_python//python/pip_install:pip_repository.bzl.
-The exception is it does not use the arg 'repo_prefix'. We set the repository
-prefix for the user and the alias arg is always True in bzlmod.
-""",
- ),
- "whl_mods": tag_class(
- attrs = _whl_mod_attrs(),
- doc = """\
-This tag class is used to create JSON file that are used when calling wheel_builder.py. These
-JSON files contain instructions on how to modify a wheel's project. Each of the attributes
-create different modifications based on the type of attribute. Previously to bzlmod these
-JSON files where referred to as annotations, and were renamed to whl_modifications in this
-extension.
-""",
- ),
- },
-)
-
-pip_internal = module_extension(
- doc = """\
-This extension is used to make dependencies from pypi available.
-
-For now this is intended to be used internally so that usage of the `pip`
-extension in `rules_python` does not affect the evaluations of the extension
-for the consumers.
-
-pip.parse:
-To use, call `pip.parse()` and specify `hub_name` and your requirements file.
-Dependencies will be downloaded and made available in a repo named after the
-`hub_name` argument.
-
-Each `pip.parse()` call configures a particular Python version. Multiple calls
-can be made to configure different Python versions, and will be grouped by
-the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
-to automatically resolve to different, Python version-specific, libraries.
-
-pip.whl_mods:
-This tag class is used to help create JSON files to describe modifications to
-the BUILD files for wheels.
-""",
- implementation = _pip_non_reproducible,
- tag_classes = {
- "override": _override_tag,
- "parse": tag_class(
- attrs = _pip_parse_ext_attrs(
- experimental_index_url = "https://pypi.org/simple",
- ),
- doc = """\
-This tag class is used to create a pypi hub and all of the spokes that are part of that hub.
-This tag class reuses most of the pypi attributes that are found in
-@rules_python//python/pip_install:pip_repository.bzl.
-The exception is it does not use the arg 'repo_prefix'. We set the repository
-prefix for the user and the alias arg is always True in bzlmod.
-""",
- ),
- "whl_mods": tag_class(
- attrs = _whl_mod_attrs(),
- doc = """\
-This tag class is used to create JSON file that are used when calling wheel_builder.py. These
-JSON files contain instructions on how to modify a wheel's project. Each of the attributes
-create different modifications based on the type of attribute. Previously to bzlmod these
-JSON files where referred to as annotations, and were renamed to whl_modifications in this
-extension.
-""",
- ),
- },
-)
-
-def _whl_mods_repo_impl(rctx):
- rctx.file("BUILD.bazel", "")
- for whl_name, mods in rctx.attr.whl_mods.items():
- rctx.file("{}.json".format(whl_name), mods)
-
-_whl_mods_repo = repository_rule(
- doc = """\
-This rule creates json files based on the whl_mods attribute.
-""",
- implementation = _whl_mods_repo_impl,
- attrs = {
- "whl_mods": attr.string_dict(
- mandatory = True,
- doc = "JSON endcoded string that is provided to wheel_builder.py",
- ),
- },
-)
+pip = pypi
+pip_internal = pypi_internal
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 79f72f0089..1a1a5a5022 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -13,6 +13,7 @@
# limitations under the License.
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
package(default_visibility = ["//:__subpackages__"])
@@ -38,6 +39,27 @@ bzl_library(
srcs = ["attrs.bzl"],
)
+bzl_library(
+ name = "bzlmod_bzl",
+ srcs = ["bzlmod.bzl"],
+ deps = [
+ ":hub_repository_bzl",
+ ":attrs_bzl",
+ ":parse_requirements_bzl",
+ ":parse_whl_name_bzl",
+ ":pip_repository_attrs_bzl",
+ ":simpleapi_download_bzl",
+ ":whl_library_bzl",
+ ":whl_repo_name_bzl",
+ "//python/private:full_version_bzl",
+ "//python/private:normalize_name_bzl",
+ "//python/private:version_label_bzl",
+ "@bazel_features//:features",
+ ] + [
+ "@pythons_hub//:interpreters_bzl",
+ ] if BZLMOD_ENABLED else [],
+)
+
bzl_library(
name = "config_settings_bzl",
srcs = ["config_settings.bzl"],
@@ -76,6 +98,16 @@ bzl_library(
],
)
+bzl_library(
+ name = "hub_repository_bzl",
+ srcs = ["hub_repository.bzl"],
+ visibility = ["//:__subpackages__"],
+ deps = [
+ ":render_pkg_aliases_bzl",
+ "//python/private:text_util_bzl",
+ ],
+)
+
bzl_library(
name = "index_sources_bzl",
srcs = ["index_sources.bzl"],
@@ -167,7 +199,7 @@ bzl_library(
"//python/private:auth_bzl",
"//python/private:normalize_name_bzl",
"//python/private:text_util_bzl",
- "//python/private/bzlmod:bazel_features_bzl",
+ "@bazel_features//:features",
],
)
diff --git a/python/private/pypi/bzlmod.bzl b/python/private/pypi/bzlmod.bzl
new file mode 100644
index 0000000000..e98208a2a6
--- /dev/null
+++ b/python/private/pypi/bzlmod.bzl
@@ -0,0 +1,818 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"pip module extension for use with bzlmod"
+
+load("@bazel_features//:features.bzl", "bazel_features")
+load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS")
+load("//python/private:auth.bzl", "AUTH_ATTRS")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:repo_utils.bzl", "repo_utils")
+load("//python/private:version_label.bzl", "version_label")
+load(":attrs.bzl", "use_isolated")
+load(":hub_repository.bzl", "hub_repository")
+load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
+load(":parse_whl_name.bzl", "parse_whl_name")
+load(":pip_repository_attrs.bzl", "ATTRS")
+load(":render_pkg_aliases.bzl", "whl_alias")
+load(":simpleapi_download.bzl", "simpleapi_download")
+load(":whl_library.bzl", "whl_library")
+load(":whl_repo_name.bzl", "whl_repo_name")
+
+def _parse_version(version):
+ major, _, version = version.partition(".")
+ minor, _, version = version.partition(".")
+ patch, _, version = version.partition(".")
+ build, _, version = version.partition(".")
+
+ return struct(
+ # use semver vocabulary here
+ major = major,
+ minor = minor,
+ patch = patch, # this is called `micro` in the Python interpreter versioning scheme
+ build = build,
+ )
+
+def _major_minor_version(version):
+ version = _parse_version(version)
+ return "{}.{}".format(version.major, version.minor)
+
+def _whl_mods_impl(mctx):
+ """Implementation of the pip.whl_mods tag class.
+
+ This creates the JSON files used to modify the creation of different wheels.
+"""
+ whl_mods_dict = {}
+ for mod in mctx.modules:
+ for whl_mod_attr in mod.tags.whl_mods:
+ if whl_mod_attr.hub_name not in whl_mods_dict.keys():
+ whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr}
+ elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys():
+ # We cannot have the same wheel name in the same hub, as we
+ # will create the same JSON file name.
+ fail("""\
+Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format(
+ whl_mod_attr.whl_name,
+ whl_mod_attr.hub_name,
+ ))
+ else:
+ whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr
+
+ for hub_name, whl_maps in whl_mods_dict.items():
+ whl_mods = {}
+
+ # create a struct that we can pass to the _whl_mods_repo rule
+ # to create the different JSON files.
+ for whl_name, mods in whl_maps.items():
+ build_content = mods.additive_build_content
+ if mods.additive_build_content_file != None and mods.additive_build_content != "":
+ fail("""\
+You cannot use both the additive_build_content and additive_build_content_file arguments at the same time.
+""")
+ elif mods.additive_build_content_file != None:
+ build_content = mctx.read(mods.additive_build_content_file)
+
+ whl_mods[whl_name] = json.encode(struct(
+ additive_build_content = build_content,
+ copy_files = mods.copy_files,
+ copy_executables = mods.copy_executables,
+ data = mods.data,
+ data_exclude_glob = mods.data_exclude_glob,
+ srcs_exclude_glob = mods.srcs_exclude_glob,
+ ))
+
+ _whl_mods_repo(
+ name = hub_name,
+ whl_mods = whl_mods,
+ )
+
+def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache):
+ logger = repo_utils.logger(module_ctx)
+ python_interpreter_target = pip_attr.python_interpreter_target
+ is_hub_reproducible = True
+
+ # if we do not have the python_interpreter set in the attributes
+ # we programmatically find it.
+ hub_name = pip_attr.hub_name
+ if python_interpreter_target == None and not pip_attr.python_interpreter:
+ python_name = "python_{}_host".format(
+ pip_attr.python_version.replace(".", "_"),
+ )
+ if python_name not in INTERPRETER_LABELS:
+ fail((
+ "Unable to find interpreter for pip hub '{hub_name}' for " +
+ "python_version={version}: Make sure a corresponding " +
+ '`python.toolchain(python_version="{version}")` call exists.' +
+ "Expected to find {python_name} among registered versions:\n {labels}"
+ ).format(
+ hub_name = hub_name,
+ version = pip_attr.python_version,
+ python_name = python_name,
+ labels = " \n".join(INTERPRETER_LABELS),
+ ))
+ python_interpreter_target = INTERPRETER_LABELS[python_name]
+
+ pip_name = "{}_{}".format(
+ hub_name,
+ version_label(pip_attr.python_version),
+ )
+ major_minor = _major_minor_version(pip_attr.python_version)
+
+ if hub_name not in whl_map:
+ whl_map[hub_name] = {}
+
+ whl_modifications = {}
+ if pip_attr.whl_modifications != None:
+ for mod, whl_name in pip_attr.whl_modifications.items():
+ whl_modifications[whl_name] = mod
+
+ if pip_attr.experimental_requirement_cycles:
+ requirement_cycles = {
+ name: [normalize_name(whl_name) for whl_name in whls]
+ for name, whls in pip_attr.experimental_requirement_cycles.items()
+ }
+
+ whl_group_mapping = {
+ whl_name: group_name
+ for group_name, group_whls in requirement_cycles.items()
+ for whl_name in group_whls
+ }
+
+ # TODO @aignas 2024-04-05: how do we support different requirement
+ # cycles for different abis/oses? For now we will need the users to
+ # assume the same groups across all versions/platforms until we start
+ # using an alternative cycle resolution strategy.
+ group_map[hub_name] = pip_attr.experimental_requirement_cycles
+ else:
+ whl_group_mapping = {}
+ requirement_cycles = {}
+
+ # Create a new wheel library for each of the different whls
+
+ get_index_urls = None
+ if pip_attr.experimental_index_url:
+ if pip_attr.download_only:
+ fail("Currently unsupported to use `download_only` and `experimental_index_url`")
+
+ get_index_urls = lambda ctx, distributions: simpleapi_download(
+ ctx,
+ attr = struct(
+ index_url = pip_attr.experimental_index_url,
+ extra_index_urls = pip_attr.experimental_extra_index_urls or [],
+ index_url_overrides = pip_attr.experimental_index_url_overrides or {},
+ sources = distributions,
+ envsubst = pip_attr.envsubst,
+ # Auth related info
+ netrc = pip_attr.netrc,
+ auth_patterns = pip_attr.auth_patterns,
+ ),
+ cache = simpleapi_cache,
+ parallel_download = pip_attr.parallel_download,
+ )
+
+ requirements_by_platform = parse_requirements(
+ module_ctx,
+ requirements_by_platform = pip_attr.requirements_by_platform,
+ requirements_linux = pip_attr.requirements_linux,
+ requirements_lock = pip_attr.requirements_lock,
+ requirements_osx = pip_attr.requirements_darwin,
+ requirements_windows = pip_attr.requirements_windows,
+ extra_pip_args = pip_attr.extra_pip_args,
+ get_index_urls = get_index_urls,
+ python_version = major_minor,
+ logger = logger,
+ )
+
+ repository_platform = host_platform(module_ctx.os)
+ for whl_name, requirements in requirements_by_platform.items():
+ # We are not using the "sanitized name" because the user
+ # would need to guess what name we modified the whl name
+ # to.
+ annotation = whl_modifications.get(whl_name)
+ whl_name = normalize_name(whl_name)
+
+ group_name = whl_group_mapping.get(whl_name)
+ group_deps = requirement_cycles.get(group_name, [])
+
+ # Construct args separately so that the lock file can be smaller and does not include unused
+ # attrs.
+ whl_library_args = dict(
+ repo = pip_name,
+ dep_template = "@{}//{{name}}:{{target}}".format(hub_name),
+ )
+ maybe_args = dict(
+ # The following values are safe to omit if they have false like values
+ annotation = annotation,
+ download_only = pip_attr.download_only,
+ enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
+ environment = pip_attr.environment,
+ envsubst = pip_attr.envsubst,
+ experimental_target_platforms = pip_attr.experimental_target_platforms,
+ group_deps = group_deps,
+ group_name = group_name,
+ pip_data_exclude = pip_attr.pip_data_exclude,
+ python_interpreter = pip_attr.python_interpreter,
+ python_interpreter_target = python_interpreter_target,
+ whl_patches = {
+ p: json.encode(args)
+ for p, args in whl_overrides.get(whl_name, {}).items()
+ },
+ )
+ whl_library_args.update({k: v for k, v in maybe_args.items() if v})
+ maybe_args_with_default = dict(
+ # The following values have defaults next to them
+ isolated = (use_isolated(module_ctx, pip_attr), True),
+ quiet = (pip_attr.quiet, True),
+ timeout = (pip_attr.timeout, 600),
+ )
+ whl_library_args.update({
+ k: v
+ for k, (v, default) in maybe_args_with_default.items()
+ if v != default
+ })
+
+ if get_index_urls:
+ # TODO @aignas 2024-05-26: move to a separate function
+ found_something = False
+ for requirement in requirements:
+ for distribution in requirement.whls + [requirement.sdist]:
+ if not distribution:
+ # sdist may be None
+ continue
+
+ found_something = True
+ is_hub_reproducible = False
+
+ if pip_attr.netrc:
+ whl_library_args["netrc"] = pip_attr.netrc
+ if pip_attr.auth_patterns:
+ whl_library_args["auth_patterns"] = pip_attr.auth_patterns
+
+ # pip is not used to download wheels and the python `whl_library` helpers are only extracting things
+ whl_library_args.pop("extra_pip_args", None)
+
+ # This is no-op because pip is not used to download the wheel.
+ whl_library_args.pop("download_only", None)
+
+ repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256)
+ whl_library_args["requirement"] = requirement.srcs.requirement
+ whl_library_args["urls"] = [distribution.url]
+ whl_library_args["sha256"] = distribution.sha256
+ whl_library_args["filename"] = distribution.filename
+ whl_library_args["experimental_target_platforms"] = requirement.target_platforms
+
+ # Pure python wheels or sdists may need to have a platform here
+ target_platforms = None
+ if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
+ if len(requirements) > 1:
+ target_platforms = requirement.target_platforms
+
+ whl_library(name = repo_name, **dict(sorted(whl_library_args.items())))
+
+ whl_map[hub_name].setdefault(whl_name, []).append(
+ whl_alias(
+ repo = repo_name,
+ version = major_minor,
+ filename = distribution.filename,
+ target_platforms = target_platforms,
+ ),
+ )
+
+ if found_something:
+ continue
+
+ requirement = select_requirement(
+ requirements,
+ platform = repository_platform,
+ )
+ if not requirement:
+ # Sometimes the package is not present for host platform if there
+ # are whls specified only in particular requirements files, in that
+ # case just continue, however, if the download_only flag is set up,
+ # then the user can also specify the target platform of the wheel
+ # packages they want to download, in that case there will be always
+ # a requirement here, so we will not be in this code branch.
+ continue
+ elif get_index_urls:
+ logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
+
+ whl_library_args["requirement"] = requirement.requirement_line
+ if requirement.extra_pip_args:
+ whl_library_args["extra_pip_args"] = requirement.extra_pip_args
+
+ # We sort so that the lock-file remains the same no matter the order of how the
+ # args are manipulated in the code going before.
+ repo_name = "{}_{}".format(pip_name, whl_name)
+ whl_library(name = repo_name, **dict(sorted(whl_library_args.items())))
+ whl_map[hub_name].setdefault(whl_name, []).append(
+ whl_alias(
+ repo = repo_name,
+ version = major_minor,
+ ),
+ )
+
+ return is_hub_reproducible
+
+def _pip_impl(module_ctx):
+ """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
+
+ This implementation iterates through all of the `pip.parse` calls and creates
+ different pip hub repositories based on the "hub_name". Each of the
+ pip calls create spoke repos that uses a specific Python interpreter.
+
+ In a MODULES.bazel file we have:
+
+ pip.parse(
+ hub_name = "pip",
+ python_version = 3.9,
+ requirements_lock = "//:requirements_lock_3_9.txt",
+ requirements_windows = "//:requirements_windows_3_9.txt",
+ )
+ pip.parse(
+ hub_name = "pip",
+ python_version = 3.10,
+ requirements_lock = "//:requirements_lock_3_10.txt",
+ requirements_windows = "//:requirements_windows_3_10.txt",
+ )
+
+ For instance, we have a hub with the name of "pip".
+ A repository named the following is created. It is actually called last when
+ all of the pip spokes are collected.
+
+ - @@rules_python~override~pip~pip
+
+ As shown in the example code above we have the following.
+ Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip".
+ These definitions create two different pip spoke repositories that are
+ related to the hub "pip".
+ One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically
+ determines the Python version and the interpreter.
+ Both of these pip spokes contain requirements files that includes websocket
+ and its dependencies.
+
+ We also need repositories for the wheels that the different pip spokes contain.
+ For each Python version a different wheel repository is created. In our example
+ each pip spoke had a requirements file that contained websockets. We
+ then create two different wheel repositories that are named the following.
+
+ - @@rules_python~override~pip~pip_39_websockets
+ - @@rules_python~override~pip~pip_310_websockets
+
+ And if the wheel has any other dependencies subsequent wheels are created in the same fashion.
+
+ The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to
+ a spoke repository depending on the Python version.
+
+ Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple
+ hubs pointing to various different pip spokes.
+
+ Some other business rules notes. A hub can only have one spoke per Python version. We cannot
+ have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second
+ we cannot have the same hub name used in sub-modules. The hub name has to be globally
+ unique.
+
+ This implementation also handles the creation of whl_modification JSON files that are used
+ during the creation of wheel libraries. These JSON files used via the annotations argument
+ when calling wheel_installer.py.
+
+ Args:
+ module_ctx: module contents
+ """
+
+ # Build all of the wheel modifications if the tag class is called.
+ _whl_mods_impl(module_ctx)
+
+ _overriden_whl_set = {}
+ whl_overrides = {}
+
+ for module in module_ctx.modules:
+ for attr in module.tags.override:
+ if not module.is_root:
+ fail("overrides are only supported in root modules")
+
+ if not attr.file.endswith(".whl"):
+ fail("Only whl overrides are supported at this time")
+
+ whl_name = normalize_name(parse_whl_name(attr.file).distribution)
+
+ if attr.file in _overriden_whl_set:
+ fail("Duplicate module overrides for '{}'".format(attr.file))
+ _overriden_whl_set[attr.file] = None
+
+ for patch in attr.patches:
+ if whl_name not in whl_overrides:
+ whl_overrides[whl_name] = {}
+
+ if patch not in whl_overrides[whl_name]:
+ whl_overrides[whl_name][patch] = struct(
+ patch_strip = attr.patch_strip,
+ whls = [],
+ )
+
+ whl_overrides[whl_name][patch].whls.append(attr.file)
+
+ # Used to track all the different pip hubs and the spoke pip Python
+ # versions.
+ pip_hub_map = {}
+
+ # Keeps track of all the hub's whl repos across the different versions.
+ # dict[hub, dict[whl, dict[version, str pip]]]
+ # Where hub, whl, and pip are the repo names
+ hub_whl_map = {}
+ hub_group_map = {}
+
+ simpleapi_cache = {}
+ is_extension_reproducible = True
+
+ for mod in module_ctx.modules:
+ for pip_attr in mod.tags.parse:
+ hub_name = pip_attr.hub_name
+ if hub_name not in pip_hub_map:
+ pip_hub_map[pip_attr.hub_name] = struct(
+ module_name = mod.name,
+ python_versions = [pip_attr.python_version],
+ )
+ elif pip_hub_map[hub_name].module_name != mod.name:
+ # We cannot have two hubs with the same name in different
+ # modules.
+ fail((
+ "Duplicate cross-module pip hub named '{hub}': pip hub " +
+ "names must be unique across modules. First defined " +
+ "by module '{first_module}', second attempted by " +
+ "module '{second_module}'"
+ ).format(
+ hub = hub_name,
+ first_module = pip_hub_map[hub_name].module_name,
+ second_module = mod.name,
+ ))
+
+ elif pip_attr.python_version in pip_hub_map[hub_name].python_versions:
+ fail((
+ "Duplicate pip python version '{version}' for hub " +
+ "'{hub}' in module '{module}': the Python versions " +
+ "used for a hub must be unique"
+ ).format(
+ hub = hub_name,
+ module = mod.name,
+ version = pip_attr.python_version,
+ ))
+ else:
+ pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
+
+ is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
+ is_extension_reproducible = is_extension_reproducible and is_hub_reproducible
+
+ for hub_name, whl_map in hub_whl_map.items():
+ hub_repository(
+ name = hub_name,
+ repo_name = hub_name,
+ whl_map = {
+ key: json.encode(value)
+ for key, value in whl_map.items()
+ },
+ default_version = _major_minor_version(DEFAULT_PYTHON_VERSION),
+ groups = hub_group_map.get(hub_name),
+ )
+
+ if bazel_features.external_deps.extension_metadata_has_reproducible:
+ # If we are not using the `experimental_index_url feature, the extension is fully
+ # deterministic and we don't need to create a lock entry for it.
+ #
+ # In order to be able to dogfood the `experimental_index_url` feature before it gets
+ # stabilized, we have created the `_pip_non_reproducible` function, that will result
+ # in extra entries in the lock file.
+ return module_ctx.extension_metadata(reproducible = is_extension_reproducible)
+ else:
+ return None
+
+def _pip_non_reproducible(module_ctx):
+ _pip_impl(module_ctx)
+
+ # We default to calling the PyPI index and that will go into the
+ # MODULE.bazel.lock file, hence return nothing here.
+ return None
+
+def _pip_parse_ext_attrs(**kwargs):
+ """Get the attributes for the pip extension.
+
+ Args:
+ **kwargs: A kwarg for setting defaults for the specific attributes. The
+ key is expected to be the same as the attribute key.
+
+ Returns:
+ A dict of attributes.
+ """
+ attrs = dict({
+ "experimental_extra_index_urls": attr.string_list(
+ doc = """\
+The extra index URLs to use for downloading wheels using bazel downloader.
+Each value is going to be subject to `envsubst` substitutions if necessary.
+
+The indexes must support Simple API as described here:
+https://packaging.python.org/en/latest/specifications/simple-repository-api/
+
+This is equivalent to `--extra-index-urls` `pip` option.
+""",
+ default = [],
+ ),
+ "experimental_index_url": attr.string(
+ default = kwargs.get("experimental_index_url", ""),
+ doc = """\
+The index URL to use for downloading wheels using bazel downloader. This value is going
+to be subject to `envsubst` substitutions if necessary.
+
+The indexes must support Simple API as described here:
+https://packaging.python.org/en/latest/specifications/simple-repository-api/
+
+In the future this could be defaulted to `https://pypi.org` when this feature becomes
+stable.
+
+This is equivalent to `--index-url` `pip` option.
+""",
+ ),
+ "experimental_index_url_overrides": attr.string_dict(
+ doc = """\
+The index URL overrides for each package to use for downloading wheels using
+bazel downloader. This value is going to be subject to `envsubst` substitutions
+if necessary.
+
+The key is the package name (will be normalized before usage) and the value is the
+index URL.
+
+This design pattern has been chosen in order to be fully deterministic about which
+packages come from which source. We want to avoid issues similar to what happened in
+https://pytorch.org/blog/compromised-nightly-dependency/.
+
+The indexes must support Simple API as described here:
+https://packaging.python.org/en/latest/specifications/simple-repository-api/
+""",
+ ),
+ "hub_name": attr.string(
+ mandatory = True,
+ doc = """
+The name of the repo pip dependencies will be accessible from.
+
+This name must be unique between modules; unless your module is guaranteed to
+always be the root module, it's highly recommended to include your module name
+in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can
+be used for shorter local names within your module.
+
+Within a module, the same `hub_name` can be specified to group different Python
+versions of pip dependencies under one repository name. This allows using a
+Python version-agnostic name when referring to pip dependencies; the
+correct version will be automatically selected.
+
+Typically, a module will only have a single hub of pip dependencies, but this
+is not required. Each hub is a separate resolution of pip dependencies. This
+means if different programs need different versions of some library, separate
+hubs can be created, and each program can use its respective hub's targets.
+Targets from different hubs should not be used together.
+""",
+ ),
+ "parallel_download": attr.bool(
+ doc = """\
+The flag allows to make use of parallel downloading feature in bazel 7.1 and above
+when the bazel downloader is used. This is by default enabled as it improves the
+performance by a lot, but in case the queries to the simple API are very expensive
+or when debugging authentication issues one may want to disable this feature.
+
+NOTE, This will download (potentially duplicate) data for multiple packages if
+there is more than one index available, but in general this should be negligible
+because the simple API calls are very cheap and the user should not notice any
+extra overhead.
+
+If we are in synchronous mode, then we will use the first result that we
+find in case extra indexes are specified.
+""",
+ default = True,
+ ),
+ "python_version": attr.string(
+ mandatory = True,
+ doc = """
+The Python version the dependencies are targetting, in Major.Minor format
+(e.g., "3.11") or patch level granularity (e.g. "3.11.1").
+
+If an interpreter isn't explicitly provided (using `python_interpreter` or
+`python_interpreter_target`), then the version specified here must have
+a corresponding `python.toolchain()` configured.
+""",
+ ),
+ "whl_modifications": attr.label_keyed_string_dict(
+ mandatory = False,
+ doc = """\
+A dict of labels to wheel names that is typically generated by the whl_modifications.
+The labels are JSON config files describing the modifications.
+""",
+ ),
+ }, **ATTRS)
+ attrs.update(AUTH_ATTRS)
+
+ return attrs
+
+def _whl_mod_attrs():
+ attrs = {
+ "additive_build_content": attr.string(
+ doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.",
+ ),
+ "additive_build_content_file": attr.label(
+ doc = """\
+(label, optional): path to a BUILD file to add to the generated
+`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file
+arguments at the same time.""",
+ ),
+ "copy_executables": attr.string_dict(
+ doc = """\
+(dict, optional): A mapping of `src` and `out` files for
+[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
+executable.""",
+ ),
+ "copy_files": attr.string_dict(
+ doc = """\
+(dict, optional): A mapping of `src` and `out` files for
+[@bazel_skylib//rules:copy_file.bzl][cf]""",
+ ),
+ "data": attr.string_list(
+ doc = """\
+(list, optional): A list of labels to add as `data` dependencies to
+the generated `py_library` target.""",
+ ),
+ "data_exclude_glob": attr.string_list(
+ doc = """\
+(list, optional): A list of exclude glob patterns to add as `data` to
+the generated `py_library` target.""",
+ ),
+ "hub_name": attr.string(
+ doc = """\
+Name of the whl modification, hub we use this name to set the modifications for
+pip.parse. If you have different pip hubs you can use a different name,
+otherwise it is best practice to just use one.
+
+You cannot have the same `hub_name` in different modules. You can reuse the same
+name in the same module for different wheels that you put in the same hub, but you
+cannot have a child module that uses the same `hub_name`.
+""",
+ mandatory = True,
+ ),
+ "srcs_exclude_glob": attr.string_list(
+ doc = """\
+(list, optional): A list of labels to add as `srcs` to the generated
+`py_library` target.""",
+ ),
+ "whl_name": attr.string(
+ doc = "The whl name that the modifications are used for.",
+ mandatory = True,
+ ),
+ }
+ return attrs
+
+# NOTE: the naming of 'override' is taken from the bzlmod native
+# 'archive_override', 'git_override' bzlmod functions.
+_override_tag = tag_class(
+ attrs = {
+ "file": attr.string(
+ doc = """\
+The Python distribution file name which needs to be patched. This will be
+applied to all repositories that setup this distribution via the pip.parse tag
+class.""",
+ mandatory = True,
+ ),
+ "patch_strip": attr.int(
+ default = 0,
+ doc = """\
+The number of leading path segments to be stripped from the file name in the
+patches.""",
+ ),
+ "patches": attr.label_list(
+ doc = """\
+A list of patches to apply to the repository *after* 'whl_library' is extracted
+and BUILD.bazel file is generated.""",
+ mandatory = True,
+ ),
+ },
+ doc = """\
+Apply any overrides (e.g. patches) to a given Python distribution defined by
+other tags in this extension.""",
+)
+
+pypi = module_extension(
+ doc = """\
+This extension is used to make dependencies from pip available.
+
+pip.parse:
+To use, call `pip.parse()` and specify `hub_name` and your requirements file.
+Dependencies will be downloaded and made available in a repo named after the
+`hub_name` argument.
+
+Each `pip.parse()` call configures a particular Python version. Multiple calls
+can be made to configure different Python versions, and will be grouped by
+the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy`
+to automatically resolve to different, Python version-specific, libraries.
+
+pip.whl_mods:
+This tag class is used to help create JSON files to describe modifications to
+the BUILD files for wheels.
+""",
+ implementation = _pip_impl,
+ tag_classes = {
+ "override": _override_tag,
+ "parse": tag_class(
+ attrs = _pip_parse_ext_attrs(),
+ doc = """\
+This tag class is used to create a pip hub and all of the spokes that are part of that hub.
+This tag class reuses most of the pip attributes that are found in
+@rules_python//python/pip_install:pip_repository.bzl.
+The exception is it does not use the arg 'repo_prefix'. We set the repository
+prefix for the user and the alias arg is always True in bzlmod.
+""",
+ ),
+ "whl_mods": tag_class(
+ attrs = _whl_mod_attrs(),
+ doc = """\
+This tag class is used to create JSON file that are used when calling wheel_builder.py. These
+JSON files contain instructions on how to modify a wheel's project. Each of the attributes
+create different modifications based on the type of attribute. Previously to bzlmod these
+JSON files where referred to as annotations, and were renamed to whl_modifications in this
+extension.
+""",
+ ),
+ },
+)
+
+pypi_internal = module_extension(
+ doc = """\
+This extension is used to make dependencies from pypi available.
+
+For now this is intended to be used internally so that usage of the `pip`
+extension in `rules_python` does not affect the evaluations of the extension
+for the consumers.
+
+pip.parse:
+To use, call `pip.parse()` and specify `hub_name` and your requirements file.
+Dependencies will be downloaded and made available in a repo named after the
+`hub_name` argument.
+
+Each `pip.parse()` call configures a particular Python version. Multiple calls
+can be made to configure different Python versions, and will be grouped by
+the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
+to automatically resolve to different, Python version-specific, libraries.
+
+pip.whl_mods:
+This tag class is used to help create JSON files to describe modifications to
+the BUILD files for wheels.
+""",
+ implementation = _pip_non_reproducible,
+ tag_classes = {
+ "override": _override_tag,
+ "parse": tag_class(
+ attrs = _pip_parse_ext_attrs(
+ experimental_index_url = "https://pypi.org/simple",
+ ),
+ doc = """\
+This tag class is used to create a pypi hub and all of the spokes that are part of that hub.
+This tag class reuses most of the pypi attributes that are found in
+@rules_python//python/pip_install:pip_repository.bzl.
+The exception is it does not use the arg 'repo_prefix'. We set the repository
+prefix for the user and the alias arg is always True in bzlmod.
+""",
+ ),
+ "whl_mods": tag_class(
+ attrs = _whl_mod_attrs(),
+ doc = """\
+This tag class is used to create JSON file that are used when calling wheel_builder.py. These
+JSON files contain instructions on how to modify a wheel's project. Each of the attributes
+create different modifications based on the type of attribute. Previously to bzlmod these
+JSON files where referred to as annotations, and were renamed to whl_modifications in this
+extension.
+""",
+ ),
+ },
+)
+
+def _whl_mods_repo_impl(rctx):
+ rctx.file("BUILD.bazel", "")
+ for whl_name, mods in rctx.attr.whl_mods.items():
+ rctx.file("{}.json".format(whl_name), mods)
+
+_whl_mods_repo = repository_rule(
+ doc = """\
+This rule creates json files based on the whl_mods attribute.
+""",
+ implementation = _whl_mods_repo_impl,
+ attrs = {
+ "whl_mods": attr.string_dict(
+ mandatory = True,
+ doc = "JSON endcoded string that is provided to wheel_builder.py",
+ ),
+ },
+)
diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/pypi/hub_repository.bzl
similarity index 79%
rename from python/private/bzlmod/pip_repository.bzl
rename to python/private/pypi/hub_repository.bzl
index 4bf3622b1b..5e209d6f9f 100644
--- a/python/private/bzlmod/pip_repository.bzl
+++ b/python/private/pypi/hub_repository.bzl
@@ -16,7 +16,7 @@
load("//python/private:text_util.bzl", "render")
load(
- "//python/private/pypi:render_pkg_aliases.bzl",
+ ":render_pkg_aliases.bzl",
"render_multiplatform_pkg_aliases",
"whl_alias",
)
@@ -28,7 +28,7 @@ package(default_visibility = ["//visibility:public"])
exports_files(["requirements.bzl"])
"""
-def _pip_repository_impl(rctx):
+def _impl(rctx):
bzl_packages = rctx.attr.whl_map.keys()
aliases = render_multiplatform_pkg_aliases(
aliases = {
@@ -66,35 +66,33 @@ def _pip_repository_impl(rctx):
"%%NAME%%": rctx.attr.repo_name,
})
-pip_repository_attrs = {
- "default_version": attr.string(
- mandatory = True,
- doc = """\
+hub_repository = repository_rule(
+ attrs = {
+ "default_version": attr.string(
+ mandatory = True,
+ doc = """\
This is the default python version in the format of X.Y. This should match
what is setup by the 'python' extension using the 'is_default = True'
setting.""",
- ),
- "groups": attr.string_list_dict(
- mandatory = False,
- ),
- "repo_name": attr.string(
- mandatory = True,
- doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.",
- ),
- "whl_map": attr.string_dict(
- mandatory = True,
- doc = """\
+ ),
+ "groups": attr.string_list_dict(
+ mandatory = False,
+ ),
+ "repo_name": attr.string(
+ mandatory = True,
+ doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.",
+ ),
+ "whl_map": attr.string_dict(
+ mandatory = True,
+ doc = """\
The wheel map where values are json.encoded strings of the whl_map constructed
in the pip.parse tag class.
""",
- ),
- "_template": attr.label(
- default = ":requirements.bzl.tmpl",
- ),
-}
-
-pip_repository = repository_rule(
- attrs = pip_repository_attrs,
+ ),
+ "_template": attr.label(
+ default = ":requirements.bzl.tmpl.bzlmod",
+ ),
+ },
doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""",
- implementation = _pip_repository_impl,
+ implementation = _impl,
)
diff --git a/python/private/bzlmod/requirements.bzl.tmpl b/python/private/pypi/requirements.bzl.tmpl.bzlmod
similarity index 100%
rename from python/private/bzlmod/requirements.bzl.tmpl
rename to python/private/pypi/requirements.bzl.tmpl.bzlmod
From ec09a4453eb35c6c2a28a199a7feb51cfad941c9 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 17:23:29 +0900
Subject: [PATCH 20/25] manually sort the bzlmod extension deps
---
python/private/pypi/BUILD.bazel | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 1a1a5a5022..1530837f7d 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -43,8 +43,8 @@ bzl_library(
name = "bzlmod_bzl",
srcs = ["bzlmod.bzl"],
deps = [
- ":hub_repository_bzl",
":attrs_bzl",
+ ":hub_repository_bzl",
":parse_requirements_bzl",
":parse_whl_name_bzl",
":pip_repository_attrs_bzl",
From e801c126bea0409b09be3561f346aa1a7eaac8ad Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 17:24:45 +0900
Subject: [PATCH 21/25] fixup
---
python/pip_install/BUILD.bazel | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 7cceb0fc9c..1894c4d915 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -73,9 +73,7 @@ filegroup(
filegroup(
name = "bzl",
- srcs = glob(["*.bzl"]) + [
- "//python/pip_install/private:bzl_srcs",
- ],
+ srcs = glob(["*.bzl"]),
visibility = ["//:__subpackages__"],
)
From 0e038dacf20b3b9913b73618e74717a19e24b137 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 17:25:19 +0900
Subject: [PATCH 22/25] Revert .bazelrc
---
.bazelrc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.bazelrc b/.bazelrc
index 22fd7057b4..07188a9196 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -4,8 +4,8 @@
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
# To update these lines, execute
# `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
test --test_output=errors
From c6f5efcfcc2b1a32041c75c0d1900de93dde92e9 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 17:31:57 +0900
Subject: [PATCH 23/25] fixup the cleanup that I thought I could do after #1987
---
python/private/pypi/whl_library.bzl | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
index d365b52154..a6344d9afa 100644
--- a/python/private/pypi/whl_library.bzl
+++ b/python/private/pypi/whl_library.bzl
@@ -29,6 +29,7 @@ load(":whl_target_platforms.bzl", "whl_target_platforms")
_CPPFLAGS = "CPPFLAGS"
_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools"
+_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point"
def _construct_pypath(rctx):
"""Helper function to construct a PYTHONPATH.
@@ -349,6 +350,30 @@ def _whl_library_impl(rctx):
metadata = json.decode(rctx.read("metadata.json"))
rctx.delete("metadata.json")
+ # NOTE @aignas 2024-06-22: this has to live on until we stop supporting
+ # passing `twine` as a `:pkg` library via the `WORKSPACE` builds.
+ #
+ # See ../../packaging.bzl line 190
+ entry_points = {}
+ for item in metadata["entry_points"]:
+ name = item["name"]
+ module = item["module"]
+ attribute = item["attribute"]
+
+ # There is an extreme edge-case with entry_points that end with `.py`
+ # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
+ entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name
+ entry_point_target_name = (
+ _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py
+ )
+ entry_point_script_name = entry_point_target_name + ".py"
+
+ rctx.file(
+ entry_point_script_name,
+ _generate_entry_point_contents(module, attribute),
+ )
+ entry_points[entry_point_without_py] = entry_point_script_name
+
build_file_contents = generate_whl_library_build_bazel(
dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
whl_name = whl_path.basename,
From 251781363e0c5c45cb145cfe0a4bce21e0bdf1ee Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 17:34:20 +0900
Subject: [PATCH 24/25] fixup! fixup the cleanup that I thought I could do
after #1987
---
python/private/pypi/whl_library.bzl | 30 ++++++++++++++++++++++++++++-
1 file changed, 29 insertions(+), 1 deletion(-)
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
index a6344d9afa..cae0db3e2b 100644
--- a/python/private/pypi/whl_library.bzl
+++ b/python/private/pypi/whl_library.bzl
@@ -386,13 +386,41 @@ def _whl_library_impl(rctx):
"pypi_name=" + metadata["name"],
"pypi_version=" + metadata["version"],
],
- entry_points = {},
+ entry_points = entry_points,
annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
)
rctx.file("BUILD.bazel", build_file_contents)
return
+def _generate_entry_point_contents(
+ module,
+ attribute,
+ shebang = "#!/usr/bin/env python3"):
+ """Generate the contents of an entry point script.
+
+ Args:
+ module (str): The name of the module to use.
+ attribute (str): The name of the attribute to call.
+ shebang (str, optional): The shebang to use for the entry point python
+ file.
+
+ Returns:
+ str: A string of python code.
+ """
+ contents = """\
+{shebang}
+import sys
+from {module} import {attribute}
+if __name__ == "__main__":
+ sys.exit({attribute}())
+""".format(
+ shebang = shebang,
+ module = module,
+ attribute = attribute,
+ )
+ return contents
+
# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique
whl_library_attrs = dict({
"annotation": attr.label(
From bd60c33b035e6236ae5dff8b9d769b1328df5fb4 Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Sat, 22 Jun 2024 18:45:04 +0900
Subject: [PATCH 25/25] fix: sort the config and annotations to produce a file
that is not modified by buildifier
---
python/private/pypi/pip_repository.bzl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl
index ab537606b1..a22f4d9d2c 100644
--- a/python/private/pypi/pip_repository.bzl
+++ b/python/private/pypi/pip_repository.bzl
@@ -191,8 +191,8 @@ def _pip_repository_impl(rctx):
p: macro_tmpl.format(p, "whl")
for p in bzl_packages
}),
- "%%ANNOTATIONS%%": render.dict(annotations),
- "%%CONFIG%%": render.dict(config),
+ "%%ANNOTATIONS%%": render.dict(dict(sorted(annotations.items()))),
+ "%%CONFIG%%": render.dict(dict(sorted(config.items()))),
"%%EXTRA_PIP_ARGS%%": json.encode(options),
"%%IMPORTS%%": "\n".join(imports),
"%%MACRO_TMPL%%": macro_tmpl,