Skip to content

Commit f8f5511

Browse files
gcurtisgcurtis
authored
patchpkg: don't error on missing store refs (#2293)
Don't exit non-zero when `devbox patch` is unable to restore some of the missing references to Python build dependencies. Fixes #2289.
1 parent 6021de4 commit f8f5511

File tree

4 files changed

+132
-30
lines changed

4 files changed

+132
-30
lines changed

‎internal/patchpkg/builder.go

Lines changed: 36 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -96,32 +96,11 @@ func (d *DerivationBuilder) Build(ctx context.Context, pkgStorePath string) erro
9696

9797
func (d *DerivationBuilder) build(ctx context.Context, pkg, out *packageFS) error {
9898
if d.RestoreRefs {
99-
// Find store path references to build inputs that were removed
100-
// from Python.
101-
refs, err := d.findRemovedRefs(ctx, pkg)
102-
if err != nil {
103-
return err
104-
}
105-
106-
// Group the references we want to restore by file path.
107-
d.bytePatches = make(map[string][]fileSlice, len(refs))
108-
for _, ref := range refs {
109-
d.bytePatches[ref.path] = append(d.bytePatches[ref.path], ref)
110-
}
111-
112-
// If any of those references have shared libraries, add them
113-
// back to Python's RPATH.
114-
if d.glibcPatcher != nil {
115-
nixStore := cmp.Or(os.Getenv("NIX_STORE"), "/nix/store")
116-
seen := make(map[string]bool)
117-
for _, ref := range refs {
118-
storePath := filepath.Join(nixStore, string(ref.data))
119-
if seen[storePath] {
120-
continue
121-
}
122-
seen[storePath] = true
123-
d.glibcPatcher.prependRPATH(newPackageFS(storePath))
124-
}
99+
if err := d.restoreMissingRefs(ctx, pkg); err != nil {
100+
// Don't break the flake build if we're unable to
101+
// restore some of the refs. Having some is still an
102+
// improvement.
103+
slog.ErrorContext(ctx, "unable to restore all removed refs", "err", err)
125104
}
126105
}
127106

@@ -152,6 +131,37 @@ func (d *DerivationBuilder) build(ctx context.Context, pkg, out *packageFS) erro
152131
return cmd.Run()
153132
}
154133

134+
func (d *DerivationBuilder) restoreMissingRefs(ctx context.Context, pkg *packageFS) error {
135+
// Find store path references to build inputs that were removed
136+
// from Python.
137+
refs, err := d.findRemovedRefs(ctx, pkg)
138+
if err != nil {
139+
return err
140+
}
141+
142+
// Group the references we want to restore by file path.
143+
d.bytePatches = make(map[string][]fileSlice, len(refs))
144+
for _, ref := range refs {
145+
d.bytePatches[ref.path] = append(d.bytePatches[ref.path], ref)
146+
}
147+
148+
// If any of those references have shared libraries, add them
149+
// back to Python's RPATH.
150+
if d.glibcPatcher != nil {
151+
nixStore := cmp.Or(os.Getenv("NIX_STORE"), "/nix/store")
152+
seen := make(map[string]bool)
153+
for _, ref := range refs {
154+
storePath := filepath.Join(nixStore, string(ref.data))
155+
if seen[storePath] {
156+
continue
157+
}
158+
seen[storePath] = true
159+
d.glibcPatcher.prependRPATH(newPackageFS(storePath))
160+
}
161+
}
162+
return nil
163+
}
164+
155165
func (d *DerivationBuilder) copyDir(out *packageFS, path string) error {
156166
path, err := out.OSPath(path)
157167
if err != nil {

‎testscripts/languages/python_patch_missing_ref.test.txt

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# Python Auto-Patch Handles Missing Ref
2+
#
3+
# Check that `devbox patch --restore-refs` doesn't break the flake build when a
4+
# a store path cannot be restored.
5+
#
6+
# The nixpkgs commit hash and version of Python chosen in this test is very
7+
# specific. Most versions don't encounter this error, so be careful that the
8+
# test still fails with Devbox v0.13.0 if changing the devbox.lock.
9+
#
10+
# https://github.com/jetify-com/devbox/issues/2289
11+
12+
exec devbox install
13+
14+
-- devbox.json --
15+
{
16+
"packages": {
17+
"python": "latest"
18+
},
19+
"env": {
20+
"PIP_DISABLE_PIP_VERSION_CHECK": "1",
21+
"PIP_NO_INPUT": "1",
22+
"PIP_NO_PYTHON_VERSION_WARNING": "1",
23+
"PIP_PROGRESS_BAR": "off",
24+
"PIP_REQUIRE_VIRTUALENV": "1",
25+
"PIP_ROOT_USER_ACTION": "ignore"
26+
},
27+
"shell": {
28+
"scripts": {
29+
"venv": ". $VENV_DIR/bin/activate && \"$@\""
30+
}
31+
}
32+
}
33+
34+
-- devbox.lock --
35+
{
36+
"lockfile_version": "1",
37+
"packages": {
38+
"python@latest": {
39+
"last_modified": "2024-09-10T15:01:03Z",
40+
"plugin_version": "0.0.4",
41+
"resolved": "github:NixOS/nixpkgs/5ed627539ac84809c78b2dd6d26a5cebeb5ae269#python3",
42+
"source": "devbox-search",
43+
"version": "3.12.5",
44+
"systems": {
45+
"aarch64-darwin": {
46+
"outputs": [
47+
{
48+
"name": "out",
49+
"path": "/nix/store/9pj4rzx5pbynkkxq1srzwjhywmcfxws3-python3-3.12.5",
50+
"default": true
51+
}
52+
],
53+
"store_path": "/nix/store/9pj4rzx5pbynkkxq1srzwjhywmcfxws3-python3-3.12.5"
54+
},
55+
"aarch64-linux": {
56+
"outputs": [
57+
{
58+
"name": "out",
59+
"path": "/nix/store/6iq3nhgdyp8a5wzwf097zf2mn4zyqxr6-python3-3.12.5",
60+
"default": true
61+
},
62+
{
63+
"name": "debug",
64+
"path": "/nix/store/xc4hygp28y7g1rvjf0vi7fj0d83a75pj-python3-3.12.5-debug"
65+
}
66+
],
67+
"store_path": "/nix/store/6iq3nhgdyp8a5wzwf097zf2mn4zyqxr6-python3-3.12.5"
68+
},
69+
"x86_64-darwin": {
70+
"outputs": [
71+
{
72+
"name": "out",
73+
"path": "/nix/store/ks8acr22s4iggnmvxydm5czl30racy32-python3-3.12.5",
74+
"default": true
75+
}
76+
],
77+
"store_path": "/nix/store/ks8acr22s4iggnmvxydm5czl30racy32-python3-3.12.5"
78+
},
79+
"x86_64-linux": {
80+
"outputs": [
81+
{
82+
"name": "out",
83+
"path": "/nix/store/h3i0acpmr8mrjx07519xxmidv8mpax4y-python3-3.12.5",
84+
"default": true
85+
},
86+
{
87+
"name": "debug",
88+
"path": "/nix/store/0a39pi2s6kxqc3kjjz2y9yzibd62zhhb-python3-3.12.5-debug"
89+
}
90+
],
91+
"store_path": "/nix/store/h3i0acpmr8mrjx07519xxmidv8mpax4y-python3-3.12.5"
92+
}
93+
}
94+
}
95+
}
96+
}

‎testscripts/languages/python_missing_so.test.txt renamed to ‎testscripts/languages/python_patch_missing_so.test.txt

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@
77
# libstdc++.so. The nixpkgs Python interpreter doesn't search standard system
88
# paths, so Devbox must patch it or provide the location of native dependencies.
99

10-
[!env:DEVBOX_RUN_FAILING_TESTS] skip 'this test doesn''t pass on Linux yet'
11-
1210
exec devbox install
1311

1412
# pip install numpy

‎testscripts/languages/python_old_glibc.test.txt renamed to ‎testscripts/languages/python_patch_old_glibc.test.txt

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
# Check that an older version of the Python interpreter (3.7) can import and run
44
# pip packages that are built from source.
55

6-
[!env:DEVBOX_RUN_FAILING_TESTS] skip 'this test doesn''t pass on Linux yet'
7-
86
exec devbox install
97

108
# pip install psycopg2

0 commit comments

Comments
 (0)