Merge remote-tracking branch 'detsys/detsys-main' into multithreaded-eval-v2

edolstra · edolstra · commit a66d268e0eaf · 2025-06-30T12:34:10.000+02:00
diff --git a/.version b/.version
@@ -1 +1 @@
-2.29.0
+2.29.1
diff --git a/.version-determinate b/.version-determinate
@@ -1 +1 @@
-3.6.6
+3.6.8
diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in
@@ -129,6 +129,8 @@
   - [Contributing](development/contributing.md)
 - [Determinate Nix Release Notes](release-notes-determinate/index.md)
   - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md)<!-- next -->
+  - [Release 3.6.8 (2025-06-25)](release-notes-determinate/rl-3.6.8.md)
+  - [Release 3.6.7 (2025-06-24)](release-notes-determinate/rl-3.6.7.md)
   - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md)
   - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md)
   - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md)
diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md
@@ -1,6 +1,6 @@
 # Changes between Nix and Determinate Nix
 
-This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.6.<!-- differences -->
+This section lists the differences between upstream Nix 2.29 and Determinate Nix 3.6.8.<!-- differences -->
 
 * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature.
 
@@ -74,3 +74,15 @@ This section lists the differences between upstream Nix 2.29 and Determinate Nix
 * Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98)
 
 <!-- Determinate Nix version 3.6.6 -->
+
+<!-- Determinate Nix version 3.6.7 -->
+
+<!-- Determinate Nix version 3.6.8 -->
+
+* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117)
+
+* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113)
+
+* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124)
+
+* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126)
diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.7.md b/doc/manual/source/release-notes-determinate/rl-3.6.7.md
@@ -0,0 +1,17 @@
+# Release 3.6.7 (2025-06-24)
+
+* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md).
+
+## What's Changed
+
+### Security contents
+
+* Patched against GHSA-g948-229j-48j3
+
+### Lazy trees:
+
+* Lazy trees now produces `flake.lock` files with NAR hashes unless `lazy-locks` is set to `true` by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113)
+* Improved caching with lazy-trees when using --impure, with enhanced testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117)
+
+
+**Full Changelog**: [v3.6.6...v3.6.7](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.7)
diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.8.md b/doc/manual/source/release-notes-determinate/rl-3.6.8.md
@@ -0,0 +1,12 @@
+# Release 3.6.8 (2025-06-25)
+
+* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md).
+
+## What's Changed
+* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117)
+* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113)
+* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124)
+* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126)
+
+
+**Full Changelog**: [v3.6.6...v3.6.8](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.8)
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
@@ -354,6 +354,8 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
 
     if (!accessor->fingerprint)
         accessor->fingerprint = result.getFingerprint(store);
+    else
+        result.cachedFingerprint = accessor->fingerprint;
 
     return {accessor, std::move(result)};
 }
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
@@ -127,8 +127,6 @@ struct PathInputScheme : InputScheme
 
         auto absPath = getAbsPath(input);
 
-        Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
-
         // FIXME: check whether access to 'path' is allowed.
         auto storePath = store->maybeParseStorePath(absPath.string());
 
@@ -137,6 +135,7 @@ struct PathInputScheme : InputScheme
 
         time_t mtime = 0;
         if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
             // FIXME: try to substitute storePath.
             auto src = sinkToSource([&](Sink & sink) {
                 mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter);
diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc
@@ -85,7 +85,6 @@ static void parseFlakeInputAttr(
 
 static FlakeInput parseFlakeInput(
     EvalState & state,
-    std::string_view inputName,
     Value * value,
     const PosIdx pos,
     const InputAttrPath & lockRootAttrPath,
@@ -155,8 +154,8 @@ static FlakeInput parseFlakeInput(
             input.ref = parseFlakeRef(state.fetchSettings, *url, {}, true, input.isFlake, true);
     }
 
-    if (!input.follows && !input.ref)
-        input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(inputName)}});
+    if (input.ref && input.follows)
+        throw Error("flake input has both a flake reference and a follows attribute, at %s", state.positions[pos]);
 
     return input;
 }
@@ -185,7 +184,6 @@ static std::pair<std::map<FlakeId, FlakeInput>, fetchers::Attrs> parseFlakeInput
         } else {
             inputs.emplace(inputName,
                 parseFlakeInput(state,
-                    inputName,
                     inputAttr.value,
                     inputAttr.pos,
                     lockRootAttrPath,
@@ -467,18 +465,27 @@ LockedFlake lockFlake(
 
             /* Get the overrides (i.e. attributes of the form
                'inputs.nixops.inputs.nixpkgs.url = ...'). */
-            for (auto & [id, input] : flakeInputs) {
+            std::function<void(const FlakeInput & input, const InputAttrPath & prefix)> addOverrides;
+            addOverrides = [&](const FlakeInput & input, const InputAttrPath & prefix)
+            {
                 for (auto & [idOverride, inputOverride] : input.overrides) {
-                    auto inputAttrPath(inputAttrPathPrefix);
-                    inputAttrPath.push_back(id);
+                    auto inputAttrPath(prefix);
                     inputAttrPath.push_back(idOverride);
-                    overrides.emplace(inputAttrPath,
-                        OverrideTarget {
-                            .input = inputOverride,
-                            .sourcePath = sourcePath,
-                            .parentInputAttrPath = inputAttrPathPrefix
-                        });
+                    if (inputOverride.ref || inputOverride.follows)
+                        overrides.emplace(inputAttrPath,
+                            OverrideTarget {
+                                .input = inputOverride,
+                                .sourcePath = sourcePath,
+                                .parentInputAttrPath = inputAttrPathPrefix
+                            });
+                    addOverrides(inputOverride, inputAttrPath);
                 }
+            };
+
+            for (auto & [id, input] : flakeInputs) {
+                auto inputAttrPath(inputAttrPathPrefix);
+                inputAttrPath.push_back(id);
+                addOverrides(input, inputAttrPath);
             }
 
             /* Check whether this input has overrides for a
@@ -534,7 +541,8 @@ LockedFlake lockFlake(
                         continue;
                     }
 
-                    assert(input.ref);
+                    if (!input.ref)
+                        input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(id)}});
 
                     auto overridenParentPath =
                         input.ref->input.isRelative()
@@ -554,7 +562,7 @@ LockedFlake lockFlake(
 
                     /* Get the input flake, resolve 'path:./...'
                        flakerefs relative to the parent flake. */
-                    auto getInputFlake = [&](const FlakeRef & ref)
+                    auto getInputFlake = [&](const FlakeRef & ref, const fetchers::UseRegistries useRegistries)
                     {
                         if (auto resolvedPath = resolveRelativePath()) {
                             return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath);
@@ -645,7 +653,7 @@ LockedFlake lockFlake(
                         }
 
                         if (mustRefetch) {
-                            auto inputFlake = getInputFlake(oldLock->lockedRef);
+                            auto inputFlake = getInputFlake(oldLock->lockedRef, useRegistriesInputs);
                             nodePaths.emplace(childNode, inputFlake.path.parent());
                             computeLocks(inputFlake.inputs, childNode, inputAttrPath, oldLock, followsPrefix,
                                 inputFlake.path, false);
@@ -670,7 +678,8 @@ LockedFlake lockFlake(
                             nuked the next time we update the lock
                             file. That is, overrides are sticky unless you
                             use --no-write-lock-file. */
-                        auto ref = (input2.ref && explicitCliOverrides.contains(inputAttrPath)) ? *input2.ref : *input.ref;
+                        auto inputIsOverride = explicitCliOverrides.contains(inputAttrPath);
+                        auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref;
 
                         /* Warn against the use of indirect flakerefs
                            (but only at top-level since we don't want
@@ -696,7 +705,7 @@ LockedFlake lockFlake(
                         };
 
                         if (input.isFlake) {
-                            auto inputFlake = getInputFlake(*input.ref);
+                            auto inputFlake = getInputFlake(*input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs);
 
                             auto childNode = make_ref<LockedNode>(
                                 inputFlake.lockedRef,
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
@@ -247,7 +247,7 @@ LocalStore::LocalStore(ref<const Config> config)
     else if (curSchema == 0) { /* new store */
         curSchema = nixSchemaVersion;
         openDB(*state, true);
-        writeFile(schemaPath, fmt("%1%", curSchema), 0666, true);
+        writeFile(schemaPath, fmt("%1%", curSchema), 0666, FsSync::Yes);
     }
 
     else if (curSchema < nixSchemaVersion) {
@@ -298,7 +298,7 @@ LocalStore::LocalStore(ref<const Config> config)
             txn.commit();
         }
 
-        writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true);
+        writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, FsSync::Yes);
 
         lockFile(globalLock.get(), ltRead, true);
     }
diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc
@@ -129,6 +129,11 @@ class DerivationBuilderImpl : public DerivationBuilder, DerivationBuilderParams
      */
     Path topTmpDir;
 
+    /**
+     * The file descriptor of the temporary directory.
+     */
+    AutoCloseFD tmpDirFd;
+
     /**
      * The path of the temporary directory in the sandbox.
      */
@@ -325,9 +330,24 @@ class DerivationBuilderImpl : public DerivationBuilder, DerivationBuilderParams
 
     /**
      * Make a file owned by the builder.
+     *
+     * SAFETY: this function is prone to TOCTOU as it receives a path and not a descriptor.
+     * It's only safe to call in a child of a directory only visible to the owner.
      */
     void chownToBuilder(const Path & path);
 
+    /**
+     * Make a file owned by the builder addressed by its file descriptor.
+     */
+    void chownToBuilder(int fd, const Path & path);
+
+    /**
+     * Create a file in `tmpDir` owned by the builder.
+     */
+    void writeBuilderFile(
+        const std::string & name,
+        std::string_view contents);
+
     /**
      * Run the builder's process.
      */
@@ -900,7 +920,14 @@ void DerivationBuilderImpl::startBuilder()
     } else {
         tmpDir = topTmpDir;
     }
-    chownToBuilder(tmpDir);
+
+    /* The TOCTOU between the previous mkdir call and this open call is unavoidable due to
+       POSIX semantics.*/
+    tmpDirFd = AutoCloseFD{open(tmpDir.c_str(), O_RDONLY | O_NOFOLLOW | O_DIRECTORY)};
+    if (!tmpDirFd)
+        throw SysError("failed to open the build temporary directory descriptor '%1%'", tmpDir);
+
+    chownToBuilder(tmpDirFd.get(), tmpDir);
 
     for (auto & [outputName, status] : initialOutputs) {
         /* Set scratch path we'll actually use during the build.
@@ -1485,9 +1512,7 @@ void DerivationBuilderImpl::initTmpDir()
             } else {
                 auto hash = hashString(HashAlgorithm::SHA256, i.first);
                 std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
+                writeBuilderFile(fn, rewriteStrings(i.second, inputRewrites));
                 env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
             }
         }
@@ -1596,11 +1621,9 @@ void DerivationBuilderImpl::writeStructuredAttrs()
 
         auto jsonSh = StructuredAttrs::writeShell(json);
 
-        writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-        chownToBuilder(tmpDir + "/.attrs.sh");
+        writeBuilderFile(".attrs.sh", rewriteStrings(jsonSh, inputRewrites));
         env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh";
-        writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-        chownToBuilder(tmpDir + "/.attrs.json");
+        writeBuilderFile(".attrs.json", rewriteStrings(json.dump(), inputRewrites));
         env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json";
     }
 }
@@ -1854,6 +1877,24 @@ void setupSeccomp()
 #endif
 }
 
+void DerivationBuilderImpl::chownToBuilder(int fd, const Path & path)
+{
+    if (!buildUser) return;
+    if (fchown(fd, buildUser->getUID(), buildUser->getGID()) == -1)
+        throw SysError("cannot change ownership of file '%1%'", path);
+}
+
+void DerivationBuilderImpl::writeBuilderFile(
+    const std::string & name,
+    std::string_view contents)
+{
+    auto path = std::filesystem::path(tmpDir) / name;
+    AutoCloseFD fd{openat(tmpDirFd.get(), name.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666)};
+    if (!fd)
+        throw SysError("creating file %s", path);
+    writeFile(fd, path, contents);
+    chownToBuilder(fd.get(), path);
+}
 
 void DerivationBuilderImpl::runChild()
 {
@@ -3065,6 +3106,15 @@ void DerivationBuilderImpl::checkOutputs(const std::map<std::string, ValidPathIn
 void DerivationBuilderImpl::deleteTmpDir(bool force)
 {
     if (topTmpDir != "") {
+        /* As an extra precaution, even in the event of `deletePath` failing to
+         * clean up, the `tmpDir` will be chowned as if we were to move
+         * it inside the Nix store.
+         *
+         * This hardens against an attack which smuggles a file descriptor
+         * to make use of the temporary directory.
+         */
+        chmod(topTmpDir.c_str(), 0000);
+
         /* Don't keep temporary directories for builtins because they
            might have privileged stuff (like a copy of netrc). */
         if (settings.keepFailed && !force && !drv.isBuiltin()) {
diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc
@@ -93,7 +93,7 @@ void restorePath(
 {
     switch (method) {
     case FileSerialisationMethod::Flat:
-        writeFile(path, source, 0666, startFsync);
+        writeFile(path, source, 0666, startFsync ? FsSync::Yes : FsSync::No);
         break;
     case FileSerialisationMethod::NixArchive:
         restorePath(path, source, startFsync);
diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc
diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh
diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc
diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh
diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh

Original file line number	Diff line number	Diff line change
`@@ -354,6 +354,8 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto`
`354`	`354`
`355`	`355`	`if (!accessor->fingerprint)`
`356`	`356`	`accessor->fingerprint = result.getFingerprint(store);`
	`357`	`+ else`
	`358`	`+ result.cachedFingerprint = accessor->fingerprint;`
`357`	`359`
`358`	`360`	`return {accessor, std::move(result)};`
`359`	`361`	`}`
Original file line number	Diff line number	Diff line change
`@@ -247,7 +247,7 @@ LocalStore::LocalStore(ref<const Config> config)`
`247`	`247`	`else if (curSchema == 0) { /* new store */`
`248`	`248`	`curSchema = nixSchemaVersion;`
`249`	`249`	`openDB(*state, true);`
`250`		`- writeFile(schemaPath, fmt("%1%", curSchema), 0666, true);`
	`250`	`+ writeFile(schemaPath, fmt("%1%", curSchema), 0666, FsSync::Yes);`
`251`	`251`	`}`
`252`	`252`
`253`	`253`	`else if (curSchema < nixSchemaVersion) {`
`@@ -298,7 +298,7 @@ LocalStore::LocalStore(ref<const Config> config)`
`298`	`298`	`txn.commit();`
`299`	`299`	`}`
`300`	`300`
`301`		`- writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true);`
	`301`	`+ writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, FsSync::Yes);`
`302`	`302`
`303`	`303`	`lockFile(globalLock.get(), ltRead, true);`
`304`	`304`	`}`