Merge pull request #52 from lucidsoftware/multiplex-sandbox

Adds multiplex sandbox support and improves argument parsing

Author: jjudd

.bazelrc_shared

@@ -14,6 +14,7 @@ build --tool_java_runtime_version="remotejdk_21"
 build --strategy=ScalaCompile=worker
 build --worker_max_instances=4
 build --worker_sandboxing
+build --experimental_worker_multiplex_sandboxing
 build --verbose_failures
 
 test --test_output=all

rules/private/phases/phase_coverage_jacoco.bzl

@@ -44,7 +44,14 @@ def phase_coverage_jacoco(ctx, g):
         outputs = [in_out_pair[1] for in_out_pair in in_out_pairs],
         executable = code_coverage_configuration.instrumentation_worker.files_to_run.executable,
         input_manifests = worker_input_manifests,
-        execution_requirements = _resolve_execution_reqs(ctx, {"supports-workers": "1"}),
+        execution_requirements = _resolve_execution_reqs(
+            ctx,
+            {
+                "supports-multiplex-workers": "1",
+                "supports-workers": "1",
+                "supports-multiplex-sandboxing": "1",
+            },
+        ),
         arguments = [args],
     )
 

rules/private/phases/phase_zinc_compile.bzl

@@ -78,13 +78,17 @@ def phase_zinc_compile(ctx, g):
     execution_requirements_tags = {
         "supports-multiplex-workers": "1",
         "supports-workers": "1",
+        "supports-multiplex-sandboxing": "1",
     }
 
-    # Disable sandboxing if incremental compilation features are going to be used
-    # because they require stashing files outside the sandbox that Bazel isn't
-    # aware of.
+    # Disable several things if incremental compilation features are going to be used
+    # because incremental compilation require stashing files outside the sandbox that
+    # Bazel isn't aware of and is less deterministic than ideal.
     if zinc_configuration.incremental:
         execution_requirements_tags["no-sandbox"] = "1"
+        execution_requirements_tags["no-cache"] = "1"
+        execution_requirements_tags["no-remote"] = "1"
+        execution_requirements_tags["supports-multiplex-sandboxing"] = "0"
 
     # todo: different execution path for nosrc jar?
     ctx.actions.run(

rules/private/phases/phase_zinc_depscheck.bzl

@@ -50,7 +50,14 @@ def phase_zinc_depscheck(ctx, g):
             outputs = [deps_check],
             executable = deps_configuration.worker.files_to_run.executable,
             input_manifests = worker_input_manifests,
-            execution_requirements = _resolve_execution_reqs(ctx, {"supports-workers": "1"}),
+            execution_requirements = _resolve_execution_reqs(
+                ctx,
+                {
+                    "supports-multiplex-workers": "1",
+                    "supports-workers": "1",
+                    "supports-multiplex-sandboxing": "1",
+                },
+            ),
             arguments = [deps_args],
         )
         deps_checks[name] = deps_check

rules/scala/private/doc.bzl

@@ -62,7 +62,14 @@ def scaladoc_implementation(ctx):
     ctx.actions.run(
         arguments = [args],
         executable = ctx.attr._runner.files_to_run.executable,
-        execution_requirements = _resolve_execution_reqs(ctx, {"supports-workers": "1"}),
+        execution_requirements = _resolve_execution_reqs(
+            ctx,
+            {
+                "supports-multiplex-workers": "1",
+                "supports-workers": "1",
+                "supports-multiplex-sandboxing": "1",
+            },
+        ),
         input_manifests = input_manifests,
         inputs = depset(
             src_jars + srcs + [zinc_configuration.compiler_bridge],

rules/scala_proto/BUILD

@@ -21,6 +21,7 @@ scala_binary(
     deps = [
         "//src/main/scala/higherkindness/rules_scala/common/args",
         "//src/main/scala/higherkindness/rules_scala/common/error",
+        "//src/main/scala/higherkindness/rules_scala/common/sandbox",
         "//src/main/scala/higherkindness/rules_scala/common/worker",
         "@annex//:net_sourceforge_argparse4j_argparse4j",
         "@annex_proto//:com_github_os72_protoc_jar",

rules/scala_proto/private/ScalaProtoWorker.scala

@@ -1,31 +1,52 @@
 package annex.scala.proto
 
+import higherkindness.rules_scala.common.args.ArgsUtil
+import higherkindness.rules_scala.common.args.ArgsUtil.PathArgumentType
 import higherkindness.rules_scala.common.args.implicits._
 import higherkindness.rules_scala.common.error.AnnexWorkerError
+import higherkindness.rules_scala.common.sandbox.SandboxUtil
 import higherkindness.rules_scala.common.worker.WorkerMain
 import java.io.{File, PrintStream}
-import java.nio.file.{Files, Paths}
+import java.nio.file.{Files, Path, Paths}
 import java.util.Collections
 import net.sourceforge.argparse4j.ArgumentParsers
 import net.sourceforge.argparse4j.impl.Arguments
-import net.sourceforge.argparse4j.inf.ArgumentParser
+import net.sourceforge.argparse4j.inf.{ArgumentParser, Namespace}
 import protocbridge.{ProtocBridge, ProtocRunner}
 import scala.jdk.CollectionConverters._
 import scalapb.ScalaPbCodeGenerator
 
 object ScalaProtoWorker extends WorkerMain[Unit] {
 
+  private class ScalaProtoRequest private (
+    val isGrpc: Boolean,
+    val outputDir: Path,
+    val protoPaths: List[Path],
+    val sources: List[Path],
+  )
+
+  private object ScalaProtoRequest {
+    def apply(workDir: Path, namespace: Namespace): ScalaProtoRequest = {
+      new ScalaProtoRequest(
+        isGrpc = namespace.getBoolean("grpc"),
+        outputDir = SandboxUtil.getSandboxPath(workDir, namespace.get[Path]("output_dir")),
+        protoPaths = SandboxUtil.getSandboxPaths(workDir, namespace.getList[Path]("proto_paths")),
+        sources = SandboxUtil.getSandboxPaths(workDir, namespace.getList[Path]("sources")),
+      )
+    }
+  }
+
   private val argParser: ArgumentParser = {
     val parser = ArgumentParsers.newFor("proto").addHelp(true).fromFilePrefix("@").build
     parser
       .addArgument("--output_dir")
       .help("Output dir")
       .metavar("output_dir")
-      .`type`(Arguments.fileType.verifyCanCreate)
+      .`type`(PathArgumentType.apply())
     parser
       .addArgument("--proto_paths")
       .nargs("*")
-      .`type`(Arguments.fileType.verifyIsDirectory)
+      .`type`(PathArgumentType.apply())
       .setDefault_(Collections.emptyList)
     parser
       .addArgument("--grpc")
@@ -35,28 +56,26 @@ object ScalaProtoWorker extends WorkerMain[Unit] {
       .help("Source files")
       .metavar("source")
       .nargs("*")
-      .`type`(Arguments.fileType.verifyCanRead.verifyIsFile)
+      .`type`(PathArgumentType.apply())
       .setDefault_(Collections.emptyList)
     parser
   }
 
   override def init(args: Option[Array[String]]): Unit = ()
 
-  protected def work(ctx: Unit, args: Array[String], out: PrintStream): Unit = {
-    val namespace = argParser.parseArgs(args)
-    val sources = namespace.getList[File]("sources").asScala.toList
-    val protoPaths = namespace.getList[File]("proto_paths").asScala.toList
+  protected def work(ctx: Unit, args: Array[String], out: PrintStream, workDir: Path): Unit = {
+    val workRequest = ScalaProtoRequest(workDir, ArgsUtil.parseArgsOrFailSafe(args, argParser, out))
 
-    val scalaOut = namespace.get[File]("output_dir").toPath
+    val scalaOut = workRequest.outputDir
     Files.createDirectories(scalaOut)
-    val outOptions = if (namespace.getBoolean("grpc")) {
+    val outOptions = if (workRequest.isGrpc) {
       "grpc:"
     } else {
       ""
     }
     val params = List(s"--scala_out=${outOptions}${scalaOut}")
-      ::: protoPaths.map(dir => s"--proto_path=${dir.toString}")
-      ::: sources.map(_.getPath.toString)
+      ::: workRequest.protoPaths.map(dir => s"--proto_path=${dir.toString}")
+      ::: workRequest.sources.map(_.toString)
 
     class MyProtocRunner[ExitCode] extends ProtocRunner[Int] {
       def run(args: Seq[String], extraEnv: Seq[(String, String)]): Int = {

rules/scala_proto/private/core.bzl

@@ -49,7 +49,14 @@ def scala_proto_library_implementation(ctx):
         tools = compiler_inputs,
         input_manifests = input_manifests,
         progress_message = "Compiling %{label} protobuf into Scala source",
-        execution_requirements = _resolve_execution_reqs(ctx, {"supports-workers": supports_workers}),
+        execution_requirements = _resolve_execution_reqs(
+            ctx,
+            {
+                "supports-multiplex-workers": supports_workers,
+                "supports-workers": supports_workers,
+                "supports-multiplex-sandboxing": supports_workers,
+            },
+        ),
         arguments = [args],
     )
 

rules/scalafmt/BUILD

@@ -24,6 +24,7 @@ scala_binary(
     scala = "//src/main/scala:bootstrap",
     visibility = ["//visibility:public"],
     deps = [
+        "//src/main/scala/higherkindness/rules_scala/common/sandbox",
         "//src/main/scala/higherkindness/rules_scala/common/worker",
         "//src/main/scala/higherkindness/rules_scala/workers/common",
         "@annex//:net_sourceforge_argparse4j_argparse4j",

rules/scalafmt/private/test.bzl

@@ -50,7 +50,14 @@ def build_format(ctx):
                 input_manifests = runner_manifests,
                 inputs = [ctx.file.config, src],
                 tools = runner_inputs,
-                execution_requirements = _resolve_execution_reqs(ctx, {"supports-multiplex-workers": "1", "supports-workers": "1"}),
+                execution_requirements = _resolve_execution_reqs(
+                    ctx,
+                    {
+                        "supports-multiplex-workers": "1",
+                        "supports-workers": "1",
+                        "supports-multiplex-sandboxing": "1",
+                    },
+                ),
                 mnemonic = "ScalaFmt",
             )
             manifest_content.append("{} {}".format(src.short_path, file.short_path))