feat: Support specifying multiple download URLs in tool_versions. (#1145)

The interface of `repository_ctx.download` and
`repository_ctx.download_and_extract` supports string lists as well as
strings as the value of the `url` argument. This is the ultimate
destination of the `url` attribute in the `tool_versions` dictionary, so
it makes sense for it to support lists as well.

It is often useful to provide multiple download URLs, e.g. when
vendoring deps through a mirror (to guard against issues like [git
archive checksums
changing](https://github.blog/changelog/2023-01-30-git-archive-checksums-may-change/)
while still keeping the canonical download URL) or in an airgapped
setting (to support internal URLs alongside external URLs). This is also
pretty common around Bazel repository rules that download things, e.g.
[http_archive](https://bazel.build/rules/lib/repo/http#http_archive-urls),
so it can be expected to work with `tool_versions` too.

Author: quval

python/repositories.bzl

@@ -99,12 +99,14 @@ def is_standalone_interpreter(rctx, python_interpreter_target):
 def _python_repository_impl(rctx):
     if rctx.attr.distutils and rctx.attr.distutils_content:
         fail("Only one of (distutils, distutils_content) should be set.")
+    if bool(rctx.attr.url) == bool(rctx.attr.urls):
+        fail("Exactly one of (url, urls) must be set.")
 
     platform = rctx.attr.platform
     python_version = rctx.attr.python_version
     python_short_version = python_version.rpartition(".")[0]
     release_filename = rctx.attr.release_filename
-    url = rctx.attr.url
+    url = rctx.attr.urls or [rctx.attr.url]
 
     if release_filename.endswith(".zst"):
         rctx.download(
@@ -428,8 +430,10 @@ For more information see the official bazel docs
             doc = "A directory prefix to strip from the extracted files.",
         ),
         "url": attr.string(
-            doc = "The URL of the interpreter to download",
-            mandatory = True,
+            doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.",
+        ),
+        "urls": attr.string_list(
+            doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.",
         ),
         "zstd_sha256": attr.string(
             default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0",
@@ -506,7 +510,7 @@ def python_register_toolchains(
         if not sha256:
             continue
 
-        (release_filename, url, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions)
+        (release_filename, urls, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions)
 
         # allow passing in a tool version
         coverage_tool = None
@@ -536,7 +540,7 @@ def python_register_toolchains(
             platform = platform,
             python_version = python_version,
             release_filename = release_filename,
-            url = url,
+            urls = urls,
             distutils = distutils,
             distutils_content = distutils_content,
             strip_prefix = strip_prefix,

python/versions.bzl

@@ -41,6 +41,8 @@ DEFAULT_RELEASE_BASE_URL = "https://github.com/indygreg/python-build-standalone/
 #       "strip_prefix": "python",
 #   },
 #
+# It is possible to provide lists in "url".
+#
 # buildifier: disable=unsorted-dict-items
 TOOL_VERSIONS = {
     "3.8.10": {
@@ -281,19 +283,28 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U
     if type(url) == type({}):
         url = url[platform]
 
+    if type(url) != type([]):
+        url = [url]
+
     strip_prefix = tool_versions[python_version].get("strip_prefix", None)
     if type(strip_prefix) == type({}):
         strip_prefix = strip_prefix[platform]
 
-    release_filename = url.format(
-        platform = platform,
-        python_version = python_version,
-        build = "shared-install_only" if (WINDOWS_NAME in platform) else "install_only",
-    )
-    if "://" in release_filename:  # is absolute url?
-        url = release_filename
-    else:
-        url = "/".join([base_url, release_filename])
+    release_filename = None
+    rendered_urls = []
+    for u in url:
+        release_filename = u.format(
+            platform = platform,
+            python_version = python_version,
+            build = "shared-install_only" if (WINDOWS_NAME in platform) else "install_only",
+        )
+        if "://" in release_filename:  # is absolute url?
+            rendered_urls.append(release_filename)
+        else:
+            rendered_urls.append("/".join([base_url, release_filename]))
+
+    if release_filename == None:
+        fail("release_filename should be set by now; were any download URLs given?")
 
     patches = tool_versions[python_version].get("patches", [])
     if type(patches) == type({}):
@@ -302,7 +313,7 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U
         else:
             patches = []
 
-    return (release_filename, url, strip_prefix, patches)
+    return (release_filename, rendered_urls, strip_prefix, patches)
 
 def print_toolchains_checksums(name):
     native.genrule(
@@ -333,10 +344,11 @@ def _commands_for_version(python_version):
         "echo \"{python_version}: {platform}: $$(curl --location --fail {release_url_sha256} 2>/dev/null || curl --location --fail {release_url} 2>/dev/null | shasum -a 256 | awk '{{ print $$1 }}')\"".format(
             python_version = python_version,
             platform = platform,
-            release_url = get_release_info(platform, python_version)[1],
-            release_url_sha256 = get_release_info(platform, python_version)[1] + ".sha256",
+            release_url = release_url,
+            release_url_sha256 = release_url + ".sha256",
         )
         for platform in TOOL_VERSIONS[python_version]["sha256"].keys()
+        for release_url in get_release_info(platform, python_version)[1]
     ])
 
 def gen_python_config_settings(name = ""):