Implement GNU Make jobserver client protocol support in Ninja

.github/workflows/linux.yml

@@ -27,6 +27,7 @@ jobs:
         run: |
           ./ninja_test --gtest_color=yes
           ../../misc/output_test.py
+          ../../misc/jobserver_test.py
       - name: Build release ninja
         run: CLICOLOR_FORCE=1 ninja -f build-Release.ninja
         working-directory: build
@@ -35,6 +36,7 @@ jobs:
         run: |
           ./ninja_test --gtest_color=yes
           ../../misc/output_test.py
+          ../../misc/jobserver_test.py
 
   build:
     runs-on: [ubuntu-latest]

CMakeLists.txt

@@ -139,6 +139,7 @@ add_library(libninja OBJECT
 	src/eval_env.cc
 	src/graph.cc
 	src/graphviz.cc
+	src/jobserver.cc
 	src/json.cc
 	src/line_printer.cc
 	src/manifest_parser.cc
@@ -156,6 +157,7 @@ if(WIN32)
 	target_sources(libninja PRIVATE
 		src/subprocess-win32.cc
 		src/includes_normalize-win32.cc
+		src/jobserver-win32.cc
 		src/msvc_helper-win32.cc
 		src/msvc_helper_main-win32.cc
 		src/getopt.c
@@ -171,7 +173,10 @@ if(WIN32)
 	# errors by telling windows.h to not define those two.
 	add_compile_definitions(NOMINMAX)
 else()
-	target_sources(libninja PRIVATE src/subprocess-posix.cc)
+	target_sources(libninja PRIVATE
+		src/jobserver-posix.cc
+		src/subprocess-posix.cc
+	)
 	if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
 		target_sources(libninja PRIVATE src/getopt.c)
 		# Build getopt.c, which can be compiled as either C or C++, as C++
@@ -277,6 +282,7 @@ if(BUILD_TESTING)
     src/elide_middle_test.cc
     src/explanations_test.cc
     src/graph_test.cc
+    src/jobserver_test.cc
     src/json_test.cc
     src/lexer_test.cc
     src/manifest_parser_test.cc

configure.py

@@ -544,6 +544,7 @@ def has_re2c() -> bool:
              'eval_env',
              'graph',
              'graphviz',
+             'jobserver',
              'json',
              'line_printer',
              'manifest_parser',
@@ -560,14 +561,17 @@ def has_re2c() -> bool:
 if platform.is_windows():
     for name in ['subprocess-win32',
                  'includes_normalize-win32',
+                 'jobserver-win32',
                  'msvc_helper-win32',
                  'msvc_helper_main-win32']:
         objs += cxx(name, variables=cxxvariables)
     if platform.is_msvc():
         objs += cxx('minidump-win32', variables=cxxvariables)
     objs += cc('getopt')
 else:
-    objs += cxx('subprocess-posix')
+    for name in ['jobserver-posix',
+                 'subprocess-posix']:
+        objs += cxx(name, variables=cxxvariables)
 if platform.is_aix():
     objs += cc('getopt')
 if platform.is_msvc():
@@ -644,6 +648,7 @@ def has_re2c() -> bool:
         'elide_middle_test',
         'explanations_test',
         'graph_test',
+        'jobserver_test',
         'json_test',
         'lexer_test',
         'manifest_parser_test',

doc/manual.asciidoc

@@ -187,10 +187,46 @@ Ninja defaults to running commands in parallel anyway, so typically
 you don't need to pass `-j`.)
 
 
+GNU Jobserver support
+~~~~~~~~~~~~~~~~~~~~~
+
+Since version 1.13., Ninja builds can follow the
+https://https://www.gnu.org/software/make/manual/html_node/Job-Slots.html[GNU Make jobserver]
+client protocol. This is useful when Ninja is invoked as part of a larger
+build system controlled by a top-level GNU Make instance, or any other
+jobserver pool implementation, as it allows better coordination between
+concurrent build tasks.
+
+This feature is automatically enabled under the following conditions:
+
+- Dry-run (i.e. `-n` or `--dry-run`) is not enabled.
+
+- No explicit job count (e.g. `-j<COUNT>`) is passed on the command
+  line.
+
+- The `MAKEFLAGS` environment variable is defined and describes a valid
+  jobserver mode using `--jobserver-auth=SEMAPHORE_NAME` on Windows, or
+  `--jobserver-auth=fifo:PATH` on Posix.
+
+In this case, Ninja will use the jobserver pool of job slots to control
+parallelism, instead of its default parallel implementation.
+
+Note that load-average limitations (i.e. when using `-l<count>`)
+are still being enforced in this mode.
+
+On Posix, Ninja supports both the `pipe` and `fifo` client modes, based on
+the content of `MAKEFLAGS`.
+
+IMPORTANT: On Posix, only the FIFO-based version of the protocol, which is
+implemented by GNU Make 4.4 and higher, is supported. Ninja will detect
+when a pipe-based jobserver is being used (i.e. when `MAKEFLAGS` contains
+`--jobserver-auth=<read>,<write>`) and will print a warning, but will
+otherwise ignore it.
+
 Environment variables
 ~~~~~~~~~~~~~~~~~~~~~
 
-Ninja supports one environment variable to control its behavior:
+Ninja supports two environment variables to control its behavior:
 `NINJA_STATUS`, the progress status printed before the rule being run.
 
 Several placeholders are available:
@@ -215,6 +251,10 @@ The default progress status is `"[%f/%t] "` (note the trailing space
 to separate from the build rule). Another example of possible progress status
 could be `"[%u/%r/%f] "`.
 
+If `MAKEFLAGS` is defined in the environment, if may alter how
+Ninja dispatches parallel build commands. See the GNU Jobserver support
+section for details.
+
 Extra tools
 ~~~~~~~~~~~