feat(venv): Static site-packages trees (#551)

This PR introduces a new suite of virtualenv machinery building on the
existing `py_binary` virtualenv tree builder to create such trees
"statically" at build time rather than "dynamically" at runtime. The
original reason this is difficult is that the `$VENV/bin/python` file is
usually a link to a static interpreter, and the interpreter specified in
`pyvenv.cfg` is also usually an absolute path to a real interpreter
outside of the venv.

The introduction of #546 or something like it makes it possible to
create a virtualenv tree which _does not_ refer to an interpreter with a
fixed path. Instead an appropriate interpreter can be dynamically
discovered by `$PATH` inspection. Support for other configuration ala
`/etc/pexrc` to facilitate interpreter discovery is possible but beyond
the scope of an initial implementation.

By using an interpreter shim, the `pyenv.cfg` file can use a relative
path to `./bin/python` as its interpreter reference, and the
`./bin/python3` and `./bin/python3.X` links can be created statically
pointing to the shim. As long as other `bin/` entrypoint scripts
reference only `#!/usr/bin/env python3` as is fairly conventional, the
constructed virtual environment will contain no absolute references to a
specific interpreter and will be fully relocatable.

A relocatable virtualenv tree containing dependencies installed into the
`site-packages` tree may be built using the new experimental `py_venv`
rule.

```
py_venv(
    name = "venv",
    deps = [
        ... # PyInfo targets to be integrated into the site-packages tree
    ],
)
```

Files are selected for inclusion into virtualenv based on import paths.
- Any import path which contains a `*.dist-info` child directory will
have its contents included.
- Any included path which ends in `site-packages` and for which there
exist files in a sibling `bin` or other conventional directory will have
those assets included into the virtualenv.

These heuristics are chosen to both handle how `rules_python` currently
emulates package installation, and to avoid dependencies on external
workspace specific behavior so that internal or vendored libraries can
be "installed" into the venv.

A `py_venv` is not intended to be a `.runfiles`-style packaging of all
source code, it is intended to be a packaging of dependency code which
is compliant with being treated as if `pip install`ed. This may not be
generally true of firstparty application code, depending on usage of
`runfiles.Runfiles` or repository-relative logics. Because of this
`py_venv` targets provide files not relocated into the venv as
`runfiles` dependencies.

Applications wishing to leverage a static virtualenv should leverage the
`py_venv_binary` rule.

```
py_venv_binary(
    name = "demo",
    srcs = ["demo.py"],
    deps = [ ... ],
    main = "demo.py",
)
```

A `py_venv_binary` defines a more conventional `.runfiles`-based Python
application, but one which will use the `.runfiles` embedded virtualenv
as the source of thirdparty packages. Firstparty packages will receive
the normal `.runfiles` workspace structure preserving treatment.

A `py_venv_test` is also provided which behaves the same as a
`py_venv_binary`.

### TODO
- [x] Get the implementation fully aligned with this writeup
- [x] Convert the existing example into a test
- [x] Create an example of using `py_venv_binary` with `rules_oci`
together to create a static application

### For follow-up work
- [x] ~Fix for #423~ Not currently possible, but we do have support when
`rules_python` catches up
- [ ] Get the zip machinery working nicely, either with a Rust impl or
via `rules_pkg`

### Changes are visible to end-users: yes

- Searched for relevant documentation and updated as needed: yes
- Breaking change (forces users to change their own code or config): no
- Suggested release notes appear below: yes

### Test plan

- New test cases added

Author: arrdem

Cargo.Bazel.lock

@@ -1,5 +1,5 @@
 {
-  "checksum": "7c8cd2683de76ab9fae29a74881389aefee00f8d83f53bd006aba44b4dd3731d",
+  "checksum": "eb371dc49c5639c18e7f0415686de673ae7f83a972b6fd8823550485f5f63bea",
   "crates": {
     "addr2line 0.24.2": {
       "name": "addr2line",
@@ -9978,14 +9978,14 @@
       ],
       "license_file": "LICENSE.txt"
     },
-    "pathdiff 0.2.2": {
+    "pathdiff 0.2.3": {
       "name": "pathdiff",
-      "version": "0.2.2",
+      "version": "0.2.3",
       "package_url": "https://github.com/Manishearth/pathdiff",
       "repository": {
         "Http": {
-          "url": "https://static.crates.io/crates/pathdiff/0.2.2/download",
-          "sha256": "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361"
+          "url": "https://static.crates.io/crates/pathdiff/0.2.3/download",
+          "sha256": "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3"
         }
       },
       "targets": [
@@ -10008,7 +10008,7 @@
           "**"
         ],
         "edition": "2018",
-        "version": "0.2.2"
+        "version": "0.2.3"
       },
       "license": "MIT/Apache-2.0",
       "license_ids": [
@@ -10856,6 +10856,14 @@
               "id": "miette 7.2.0",
               "target": "miette"
             },
+            {
+              "id": "pathdiff 0.2.3",
+              "target": "pathdiff"
+            },
+            {
+              "id": "relative-path 1.9.3",
+              "target": "relative_path"
+            },
             {
               "id": "tempfile 3.13.0",
               "target": "tempfile"
@@ -10891,6 +10899,10 @@
             {
               "id": "uv-virtualenv 0.0.4",
               "target": "uv_virtualenv"
+            },
+            {
+              "id": "walkdir 2.5.0",
+              "target": "walkdir"
             }
           ],
           "selects": {}
@@ -12203,6 +12215,51 @@
       ],
       "license_file": "LICENSE-APACHE"
     },
+    "relative-path 1.9.3": {
+      "name": "relative-path",
+      "version": "1.9.3",
+      "package_url": "https://github.com/udoprog/relative-path",
+      "repository": {
+        "Http": {
+          "url": "https://static.crates.io/crates/relative-path/1.9.3/download",
+          "sha256": "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+        }
+      },
+      "targets": [
+        {
+          "Library": {
+            "crate_name": "relative_path",
+            "crate_root": "src/lib.rs",
+            "srcs": {
+              "allow_empty": true,
+              "include": [
+                "**/*.rs"
+              ]
+            }
+          }
+        }
+      ],
+      "library_target_name": "relative_path",
+      "common_attrs": {
+        "compile_data_glob": [
+          "**"
+        ],
+        "crate_features": {
+          "common": [
+            "default"
+          ],
+          "selects": {}
+        },
+        "edition": "2021",
+        "version": "1.9.3"
+      },
+      "license": "MIT OR Apache-2.0",
+      "license_ids": [
+        "Apache-2.0",
+        "MIT"
+      ],
+      "license_file": null
+    },
     "rend 0.5.2": {
       "name": "rend",
       "version": "0.5.2",
@@ -21868,7 +21925,7 @@
               "target": "mailparse"
             },
             {
-              "id": "pathdiff 0.2.2",
+              "id": "pathdiff 0.2.3",
               "target": "pathdiff"
             },
             {
@@ -22997,7 +23054,7 @@
               "target": "itertools"
             },
             {
-              "id": "pathdiff 0.2.2",
+              "id": "pathdiff 0.2.3",
               "target": "pathdiff"
             },
             {
@@ -27568,6 +27625,8 @@
     "clap 4.5.20",
     "itertools 0.13.0",
     "miette 7.2.0",
+    "pathdiff 0.2.3",
+    "relative-path 1.9.3",
     "tempfile 3.13.0",
     "thiserror 1.0.65",
     "uv-cache 0.0.1",
@@ -27576,7 +27635,8 @@
     "uv-install-wheel 0.0.1",
     "uv-pypi-types 0.0.1",
     "uv-python 0.0.1",
-    "uv-virtualenv 0.0.4"
+    "uv-virtualenv 0.0.4",
+    "walkdir 2.5.0"
   ],
   "direct_dev_deps": []
 }

Cargo.lock

@@ -24,7 +24,7 @@ bazel_lib.expand_template()
 python = use_extension("@rules_python//python/extensions:python.bzl", "python", dev_dependency = True)
 python.toolchain(
     is_default = False,
-    python_version = "3.8.12",
+    python_version = "3.9",
 )
 
 tools = use_extension("//py:extensions.bzl", "py_tools")
@@ -88,7 +88,13 @@ crate.from_cargo(
 use_repo(crate, "crate_index")
 
 # For building test images with py_image_layer
-bazel_dep(name = "container_structure_test", version = "1.19.1", dev_dependency = True)
+bazel_dep(name = "container_structure_test", version = "1.19.3", dev_dependency = True)
+git_override(
+    module_name = "container_structure_test",
+    commit = "56c7201716d770c0f820a9c19207ba2ea77c34f8",
+    remote = "https://github.com/GoogleContainerTools/container-structure-test.git",
+)
+
 bazel_dep(name = "rules_oci", version = "2.0.1", dev_dependency = True)
 
 oci = use_extension("@rules_oci//oci:extensions.bzl", "oci", dev_dependency = True)

MODULE.bazel

@@ -38,7 +38,7 @@ stardoc_with_diff_test(
 
 stardoc_with_diff_test(
     name = "venv",
-    bzl_library_target = "//py/private:py_venv",
+    bzl_library_target = "//py/private/py_venv:py_venv",
 )
 
 update_docs(name = "update")