diff --git a/.drone.jsonnet b/.drone.jsonnet index 541220b794..7a9eb337e5 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -685,7 +685,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') platform: { arch: arch }, // [if arch == 'arm64' then 'node']: { arch: 'arm64' }, clone: { depth: 10 }, - steps: [ + steps: std.filter(function(step) step != null, [ { name: 'submodules', image: 'alpine/git', @@ -717,9 +717,41 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'cp -r /drone/src /mdb/' + builddir + '/storage/columnstore/columnstore', ], }, + { + // Get the copy of compile cache from the nightly build from S3 + name: 'get-local-sccache', + depends_on: ['submodules'], + image: 'amazon/aws-cli', + volumes: [pipeline._volumes.mdb], + environment: { + AWS_ACCESS_KEY_ID: { + from_secret: 'aws_access_key_id', + }, + AWS_SECRET_ACCESS_KEY: { + from_secret: 'aws_secret_access_key', + }, + }, + commands: [ + 'yum install -y tar zstd', + 'mkdir -p /mdb/sccache', + 'echo "Attempting to download SCCache archive from S3..."', + 'if aws s3 cp s3://cspkg/nightly-sccache/stable-23.10-' + server + '-' + arch + '-' + std.strReplace(platform, ":", "-") + '/sccache.tar.zst /tmp/sccache.tar.zst; then', + ' echo "SCCache archive found. Extracting..."', + ' tar -I zstd -xf /tmp/sccache.tar.zst -C /mdb/sccache', + + // TODO remove when all correct caches are uploaded + ' ls /mdb/sccache/; if [ -f /mdb/sccache/mdb ]; then mv /mdb/sccache/mdb/sccache /mdb/sccache_inner && rm -rf /mdb/sccache/ && mv /mdb/sccache_inner /mdb/sccache; fi', + + ' echo "SCCache successfully downloaded and extracted."', + ' ls /mdb/sccache', + 'else', + ' echo "SCCache archive not found on S3. Proceeding without it."', + 'fi', + ], + }, { name: 'build', - depends_on: ['clone-mdb'], + depends_on: ['clone-mdb', 'get-local-sccache'], image: img, volumes: [pipeline._volumes.mdb], environment: { @@ -733,16 +765,15 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') AWS_SECRET_ACCESS_KEY: { from_secret: 'aws_secret_access_key', }, - SCCACHE_BUCKET: 'cs-sccache', - SCCACHE_REGION: 'us-east-1', - SCCACHE_S3_USE_SSL: 'true', - SCCACHE_S3_KEY_PREFIX: result + branch + server + arch + '${DRONE_PULL_REQUEST}', - //SCCACHE_ERROR_LOG: '/tmp/sccache_log.txt', - //SCCACHE_LOG: 'debug', + SCCACHE_DIR: '/mdb/sccache', // local cache for sccache + SCCACHE_CACHE_SIZE: '20G', + // SCCACHE_ERROR_LOG: '/tmp/sccache_log.txt', + // SCCACHE_LOG: 'sccache_debug.txt', }, commands: [ 'export CLICOLOR_FORCE=1', get_sccache, + 'mkdir -p /mdb/sccache', 'mkdir /mdb/' + builddir + '/' + result, 'bash -c "set -o pipefail && bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/bootstrap_mcs.sh ' + @@ -752,7 +783,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') '--server-version ' + server + ' | ' + '/mdb/' + builddir + '/storage/columnstore/columnstore/build/ansi2txt.sh ' + '/mdb/' + builddir + '/' + result + '/build.log"' , - 'sccache --show-stats', + 'sccache --show-adv-stats', // move engine and cmapi packages to one dir and make a repo if (pkg_format == 'rpm') then "mv -v -t ./" + result + "/ /mdb/" + builddir + "/*.rpm /drone/src/cmapi/" + result + "/*.rpm && createrepo ./" + result @@ -762,6 +793,30 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager', ], }, + // (if (event == 'cron') then { // Publish cache only in the nightly builds) + (if (1 == 1) then { + name: 'publish-sccache', + depends_on: ['build'], + image: 'amazon/aws-cli', + volumes: [pipeline._volumes.mdb], + environment: { + AWS_ACCESS_KEY_ID: { + from_secret: 'aws_access_key_id', + }, + AWS_SECRET_ACCESS_KEY: { + from_secret: 'aws_secret_access_key', + }, + }, + commands: [ + 'yum install -y tar zstd', + 'du -hs /mdb/sccache', + 'cd /mdb/sccache', + 'time tar -I pzstd -cf ../sccache.tar.zst .', + 'cd ..', + 'time aws s3 cp sccache.tar.zst s3://cspkg/nightly-sccache/stable-23.10-' + server + '-' + arch + '-' + std.strReplace(platform, ':', '-') + '/sccache.tar.zst', + 'echo "Nightly build cache uploaded to: s3://cspkg/nightly-sccache/stable-23.10-' + server + '-' + arch + '-' + std.strReplace(platform, ':', '-') + '/sccache.tar.zst"', + ], + } else null), { name: 'unittests', depends_on: ['build'], @@ -804,7 +859,7 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise') 'ls -l /drone/src/%s | grep columnstore' % result, ], }, - ] + + ]) + [pipeline.cmapipython] + [pipeline.cmapibuild] + [pipeline.publish('cmapi build')] + [pipeline.publish()] + diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 7f4a8fd12d..364684fc67 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -317,6 +317,11 @@ construct_cmake_flags() { if [[ $SCCACHE = true ]]; then warn "Use sccache" + # If sccache works well, the build becomes an IO bound task instead of CPU bound + # It happens because most of the time is spent is hashing the input files + # So we can run more jobs in parallel + CPUS=$((CPUS * 3)) + message "Adjusted the number of CPUs for parallel build to $CPUS (multiplied by 3 to reduce IO wait)" MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" fi