Add external builders

These are helper programs that execute derivations for specified
system types (e.g. using QEMU to emulate another system type).

To use, set `external-builders`:

  external-builders = [{"systems": ["aarch64-linux"], "program": "/path/to/external-builder.py"}]

The external builder gets one command line argument, the path to a JSON file containing all necessary information about the derivation:

  {
    "args": [...],
    "builder": "/nix/store/kwcyvgdg98n98hqapaz8sw92pc2s78x6-bash-5.2p37/bin/bash",
    "env": {
      "HOME": "/homeless-shelter",
      ...
    },
    "realStoreDir": "/tmp/nix/nix/store",
    "storeDir": "/nix/store",
    "tmpDir": "/tmp/nix-shell.dzQ2hE/nix-build-patchelf-0.14.3.drv-46/build",
    "tmpDirInSandbox": "/build"
  }

Author: edolstra

src/libstore/globals.cc

@@ -309,6 +309,17 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const
     }
 }
 
+NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Settings::ExternalBuilder, systems, program);
+
+template<> Settings::ExternalBuilders BaseSetting<Settings::ExternalBuilders>::parse(const std::string & str) const
+{
+    return nlohmann::json::parse(str).template get<Settings::ExternalBuilders>();
+}
+
+template<> std::string BaseSetting<Settings::ExternalBuilders>::to_string() const
+{
+    return nlohmann::json(value).dump();
+}
 
 static void preloadNSS()
 {

src/libstore/include/nix/store/globals.hh

@@ -1237,6 +1237,23 @@ public:
           Set it to 1 to warn on all paths.
         )"
     };
+
+    struct ExternalBuilder
+    {
+        std::vector<std::string> systems;
+        Path program;
+    };
+
+    using ExternalBuilders = std::vector<ExternalBuilder>;
+
+    Setting<ExternalBuilders> externalBuilders{
+        this,
+        {},
+        "external-builders",
+        R"(
+          Helper programs that execute derivations.
+        )"
+    };
 };
 
 

src/libstore/unix/build/derivation-builder.cc

@@ -208,6 +208,12 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder
         return acquireUserLock(1, false);
     }
 
+    /**
+     * Throw an exception if we can't do this derivation because of
+     * missing system features.
+     */
+    virtual void checkSystem();
+
     /**
      * Return the paths that should be made available in the sandbox.
      * This includes:
@@ -675,13 +681,8 @@ static void handleChildException(bool sendException)
     }
 }
 
-void DerivationBuilderImpl::startBuilder()
+void DerivationBuilderImpl::checkSystem()
 {
-    /* Make sure that no other processes are executing under the
-       sandbox uids. This must be done before any chownToBuilder()
-       calls. */
-    prepareUser();
-
     /* Right platform? */
     if (!drvOptions.canBuildLocally(store, drv)) {
         auto msg = fmt(
@@ -701,6 +702,16 @@ void DerivationBuilderImpl::startBuilder()
 
         throw BuildError(msg);
     }
+}
+
+void DerivationBuilderImpl::startBuilder()
+{
+    checkSystem();
+
+    /* Make sure that no other processes are executing under the
+       sandbox uids. This must be done before any chownToBuilder()
+       calls. */
+    prepareUser();
 
     /* Create a temporary directory where the build will take
        place. */
@@ -2110,6 +2121,7 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path)
 // FIXME: do this properly
 #include "linux-derivation-builder.cc"
 #include "darwin-derivation-builder.cc"
+#include "external-derivation-builder.cc"
 
 namespace nix {
 
@@ -2118,6 +2130,9 @@ std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
     std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
     DerivationBuilderParams params)
 {
+    if (auto builder = ExternalDerivationBuilder::newIfSupported(store, miscMethods, params))
+        return builder;
+
     bool useSandbox = false;
 
     /* Are we doing a sandboxed build? */

src/libstore/unix/build/external-derivation-builder.cc

@@ -0,0 +1,107 @@
+namespace nix {
+
+struct ExternalDerivationBuilder : DerivationBuilderImpl
+{
+    Settings::ExternalBuilder externalBuilder;
+
+    ExternalDerivationBuilder(
+        Store & store,
+        std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
+        DerivationBuilderParams params,
+        Settings::ExternalBuilder externalBuilder)
+        : DerivationBuilderImpl(store, std::move(miscMethods), std::move(params))
+        , externalBuilder(std::move(externalBuilder))
+    {
+    }
+
+    static std::unique_ptr<ExternalDerivationBuilder> newIfSupported(
+        Store & store, std::unique_ptr<DerivationBuilderCallbacks> & miscMethods, DerivationBuilderParams & params)
+    {
+        for (auto & handler : settings.externalBuilders.get()) {
+            for (auto & system : handler.systems)
+                if (params.drv.platform == system)
+                    return std::make_unique<ExternalDerivationBuilder>(
+                        store, std::move(miscMethods), std::move(params), std::move(handler));
+        }
+        return {};
+    }
+
+    bool prepareBuild() override
+    {
+        // External builds don't use build users, so this always
+        // succeeds.
+        return true;
+    }
+
+    Path tmpDirInSandbox() override
+    {
+        /* In a sandbox, for determinism, always use the same temporary
+           directory. */
+        return "/build";
+    }
+
+    void setBuildTmpDir() override
+    {
+        tmpDir = topTmpDir + "/build";
+        createDir(tmpDir, 0700);
+    }
+
+    void prepareUser() override
+    {
+        // Nothing to do here since we don't have a build user.
+    }
+
+    void checkSystem() override
+    {
+        // FIXME: should check system features.
+    }
+
+    void startChild() override
+    {
+        if (drvOptions.getRequiredSystemFeatures(drv).count("recursive-nix"))
+            throw Error("'recursive-nix' is not supported yet by external derivation builders");
+
+        auto json = nlohmann::json::object();
+
+        json.emplace("builder", drv.builder);
+        {
+            auto l = nlohmann::json::array();
+            for (auto & i : drv.args)
+                l.push_back(rewriteStrings(i, inputRewrites));
+            json.emplace("args", std::move(l));
+        }
+        {
+            auto j = nlohmann::json::object();
+            for (auto & [name, value] : env)
+                j.emplace(name, rewriteStrings(value, inputRewrites));
+            json.emplace("env", std::move(j));
+        }
+        json.emplace("topTmpDir", topTmpDir);
+        json.emplace("tmpDir", tmpDir);
+        json.emplace("tmpDirInSandbox", tmpDirInSandbox());
+        json.emplace("storeDir", store.storeDir);
+        json.emplace("realStoreDir", getLocalStore(store).config->realStoreDir.get());
+        json.emplace("system", drv.platform);
+
+        auto jsonFile = topTmpDir + "/build.json";
+        writeFile(jsonFile, json.dump());
+
+        pid = startProcess([&]() {
+            openSlave();
+            try {
+                commonChildInit();
+
+                Strings args = {externalBuilder.program, jsonFile};
+
+                execv(externalBuilder.program.c_str(), stringsToCharPtrs(args).data());
+
+                throw SysError("executing '%s'", externalBuilder.program);
+            } catch (...) {
+                handleChildException(true);
+                _exit(1);
+            }
+        });
+    }
+};
+
+}