Merge pull request #4458 from kinke/custom_ninja

CI: Switch to custom ninja, trying to schedule `ldc2.o` compilation early

Author: kinke

.circleci/config.yml

@@ -18,19 +18,31 @@ commonSteps: &commonSteps
             fi
             apt-get -q update
             apt-get -yq install \
-              git-core cmake ninja-build $gcc_pkg \
+              git-core $gcc_pkg \
               zlib1g-dev $libcurl_pkg curl gdb python3 python3-pip tzdata unzip zip \
               $EXTRA_APT_PACKAGES
+            # Download & extract CMake
+            curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-linux-x86_64.tar.gz
+            mkdir cmake
+            tar -xf cmake.tar.gz --strip 1 -C cmake
+            rm cmake.tar.gz
+            # Download & extract Ninja
+            curl -fL --retry 3 --max-time 60 -O https://github.com/symmetryinvestments/ninja/releases/download/v1.11.1-sym1/ninja-linux.zip
+            mkdir ninja
+            unzip ninja-linux.zip -d ninja
+            rm ninja-linux.zip
+            # Add CMake and Ninja to PATH for future steps
+            echo "export PATH=$PWD/cmake/bin:$PWD/ninja:$PATH" >> $BASH_ENV
           else
             # Download & extract CMake
-            curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-macos-universal.tar.gz
+            curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-macos-universal.tar.gz
             mkdir cmake
             tar -xf cmake.tar.gz --strip 3 -C cmake
             rm cmake.tar.gz
             # Download & extract Ninja
-            curl -fL --retry 3 --max-time 60 -O https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-mac.zip
+            curl -fL --retry 3 --max-time 60 -O https://github.com/symmetryinvestments/ninja/releases/download/v1.11.1-sym1/ninja-mac.zip
             mkdir ninja
-            tar -xf ninja-mac.zip -C ninja
+            unzip ninja-mac.zip -d ninja
             rm ninja-mac.zip
             # Download & extract LDC-flavoured LLVM with enabled assertions
             curl -fL --retry 3 --max-time 300 -o llvm.tar.xz https://github.com/ldc-developers/llvm-project/releases/download/ldc-v$LLVM_VERSION/llvm-$LLVM_VERSION-osx-x86_64-withAsserts.tar.xz
@@ -68,12 +80,7 @@ commonSteps: &commonSteps
             -DD_COMPILER=$PWD/../host-ldc/bin/ldmd2 \
             -DLDC_LINK_MANUALLY=OFF \
             $EXTRA_CMAKE_FLAGS
-          # Work around out-of-memory errors - retry once with parallelization and one last time serially
-          targets='all ldc2-unittest all-test-runners'
-          for i in {1..2}; do
-            ninja -j$PARALLELISM -k0 $targets && break || true
-          done
-          ninja -j1 $targets
+          ninja -j$PARALLELISM obj/ldc2.o all ldc2-unittest all-test-runners
           bin/ldc2 -version
     - run:
         name: Run LDC D unittests

.cirrus.yml

@@ -27,7 +27,7 @@ common_steps_template: &COMMON_STEPS_TEMPLATE
       -DINCLUDE_INSTALL_DIR=$installDir/import \
       -DLDC_LINK_MANUALLY=OFF \
       "${extraFlags[@]}"
-    ninja -j$PARALLELISM all ldc2-unittest all-test-runners
+    ninja -j$PARALLELISM obj/ldc2.o all ldc2-unittest all-test-runners
     bin/ldc2 -version
   always:
     run_compiler_unittests_script: |
@@ -190,10 +190,22 @@ install_ubuntu_prerequisites_template: &INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
     fi
     apt-get -q update
     apt-get -yq install \
-      git-core cmake ninja-build $gcc_pkg \
+      git-core ninja-build $gcc_pkg \
       zlib1g-dev $libcurl_pkg curl gdb python3 python3-pip tzdata unzip zip \
       $EXTRA_APT_PACKAGES
     python3 --version
+    # Download & extract CMake
+    curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-linux-$CI_ARCH.tar.gz
+    mkdir cmake
+    tar -xf cmake.tar.gz --strip 1 -C cmake
+    rm cmake.tar.gz
+    # Download & extract custom Ninja on x86_64
+    if [[ "$CI_ARCH" == "x86_64" ]]; then
+      curl -fL --retry 3 --max-time 60 -O https://github.com/symmetryinvestments/ninja/releases/download/v1.11.1-sym1/ninja-linux.zip
+      mkdir ninja
+      unzip ninja-linux.zip -d ninja
+      rm ninja-linux.zip
+    fi
     # Download & extract host LDC
     curl -fL --retry 3 --max-time 300 -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-linux-$CI_ARCH.tar.xz
     mkdir host-ldc
@@ -207,12 +219,12 @@ install_macos_prerequisites_template: &INSTALL_MACOS_PREREQUISITES_TEMPLATE
     cd $CIRRUS_WORKING_DIR/..
     sysctl -n hw.logicalcpu
     # Download & extract CMake
-    curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-macos-universal.tar.gz
+    curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-macos-universal.tar.gz
     mkdir cmake
     tar -xf cmake.tar.gz --strip 3 -C cmake
     rm cmake.tar.gz
     # Download & extract Ninja
-    curl -fL --retry 3 --max-time 60 -O https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-mac.zip
+    curl -fL --retry 3 --max-time 60 -O https://github.com/symmetryinvestments/ninja/releases/download/v1.11.1-sym1/ninja-mac.zip
     mkdir ninja
     tar -xf ninja-mac.zip -C ninja
     rm ninja-mac.zip
@@ -253,6 +265,7 @@ task:
     EXTRA_APT_PACKAGES: "llvm-dev libclang-common-10-dev"
     EXTRA_CMAKE_FLAGS: "-DMULTILIB=ON -DRT_SUPPORT_SANITIZERS=ON -DBUILD_LTO_LIBS=ON"
     PARALLELISM: 8
+    PATH: ${CIRRUS_WORKING_DIR}/../cmake/bin:${CIRRUS_WORKING_DIR}/../ninja:${PATH}
   << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
   << : *COMMON_STEPS_TEMPLATE
 
@@ -271,6 +284,7 @@ task:
     EXTRA_APT_PACKAGES: "gdmd llvm-dev libclang-common-15-dev"
     EXTRA_CMAKE_FLAGS: "-DBUILD_SHARED_LIBS=ON -DBUILD_LTO_LIBS=ON -DD_COMPILER=gdmd -DLDC_LINK_MANUALLY=ON"
     PARALLELISM: 8
+    PATH: ${CIRRUS_WORKING_DIR}/../cmake/bin:${CIRRUS_WORKING_DIR}/../ninja:${PATH}
     # for gdmd:
     LANG: C.UTF-8
   << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
@@ -290,6 +304,7 @@ task:
     EXTRA_APT_PACKAGES: "llvm-dev libclang-common-10-dev"
     EXTRA_CMAKE_FLAGS: "-DBUILD_LTO_LIBS=ON"
     PARALLELISM: 8
+    PATH: ${CIRRUS_WORKING_DIR}/../cmake/bin:${CIRRUS_WORKING_DIR}/../ninja:${PATH}
   << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
   << : *COMMON_STEPS_TEMPLATE
 
@@ -347,6 +362,7 @@ task:
       -DLLVM_ROOT_DIR=$CIRRUS_WORKING_DIR/../llvm
       -DD_COMPILER=$CIRRUS_WORKING_DIR/../bootstrap-ldc/bin/ldmd2
     PARALLELISM: 4
+    PATH: ${CIRRUS_WORKING_DIR}/../cmake/bin:${PATH}
     CLANG_VERSION: '15.0.3' # 15.0.6 requires a more recent libstdc++.so.6 than shipped with Ubuntu 20
     CC: $CIRRUS_WORKING_DIR/../clang/bin/clang
     CXX: $CIRRUS_WORKING_DIR/../clang/bin/clang++
@@ -390,7 +406,7 @@ task:
       -DD_COMPILER=$PWD/../host-ldc/bin/ldmd2 \
       -DBUILD_SHARED_LIBS=OFF \
       -DBUILD_LTO_LIBS=ON
-    ninja -j$PARALLELISM
+    ninja -j$PARALLELISM obj/ldc2.o all
     bin/ldc2 -version
   << : *COMMON_STEPS_TEMPLATE
   << : *PACKAGING_STEPS_TEMPLATE

.github/actions/1-setup/action.yml

@@ -103,7 +103,7 @@ runs:
         ld --version
 
     - name: Install ninja
-      uses: seanmiddleditch/gha-setup-ninja@v3
+      uses: symmetryinvestments/gha-setup-ninja@v1
 
     - name: Install D host compiler
       uses: dlang-community/setup-dlang@v1

.github/actions/helper-build-ldc/action.yml

@@ -15,7 +15,7 @@ inputs:
     default: ''
   build_targets:
     required: false
-    default: '' # all
+    default: 'all'
   arch:
     required: false # Windows only
 runs:
@@ -39,7 +39,7 @@ runs:
           ${{ inputs.specify_install_dir == 'true' && '-DINCLUDE_INSTALL_DIR="$installDir/import"' || '' }} \
           ${{ inputs.cmake_flags }}
 
-        ninja ${{ inputs.build_targets }}
+        ninja obj/ldc2.o ${{ inputs.build_targets }}
 
     # Windows: invoke CMake & ninja in MSVC env
     - if: runner.os == 'Windows'
@@ -62,4 +62,4 @@ runs:
           ${{ inputs.cmake_flags }}
         if %errorlevel% neq 0 exit /b %errorlevel%
 
-        ninja ${{ inputs.build_targets }} || exit /b
+        ninja obj/ldc2.obj ${{ inputs.build_targets }} || exit /b

.github/workflows/supported_llvm_versions.yml

@@ -58,7 +58,7 @@ jobs:
           submodules: true
           fetch-depth: 50
       - name: Install ninja
-        uses: seanmiddleditch/gha-setup-ninja@v3
+        uses: symmetryinvestments/gha-setup-ninja@v1
       - name: Install D host compiler
         uses: dlang-community/setup-dlang@v1
         with:
@@ -122,7 +122,7 @@ jobs:
             -DLLVM_ROOT_DIR="$PWD/llvm" \
             -DLDC_LINK_MANUALLY=OFF \
             ${{ matrix.cmake_flags }}
-          ninja all ldc2-unittest all-test-runners
+          ninja obj/ldc2.o all ldc2-unittest all-test-runners
           bin/ldc2 --version
 
       - name: Run LDC D unittests

CMakeLists.txt

@@ -1,6 +1,12 @@
 cmake_minimum_required(VERSION 3.4.3)
 if(POLICY CMP0025)
-  cmake_policy(SET CMP0025 NEW)
+    cmake_policy(SET CMP0025 NEW)
+endif()
+if(${CMAKE_VERSION} VERSION_GREATER "3.26.9")
+    # Prevent implicit dependencies for custom commands, e.g.,
+    # `obj/ldc2.o` depending on `lib/libldc.a` with LDC_LINK_MANUALLY=ON.
+    # Only supported since CMake v3.27 unfortunately.
+    set(CMAKE_ADD_CUSTOM_COMMAND_DEPENDS_EXPLICIT_ONLY ON)
 endif()
 
 project(ldc)

runtime/CMakeLists.txt

@@ -335,6 +335,7 @@ if(TARGET gen_gccbuiltins)
     add_custom_command(
         OUTPUT ${module}
         COMMAND gen_gccbuiltins ${module} "${name}"
+        DEPENDS gen_gccbuiltins
     )
   endfunction()