Merge branch 'main' of github.com:ClickHouse/ClickBench

Author: alexey-milovidov

.github/workflows/clickhouse-cloud.yml

@@ -1,6 +1,11 @@
 name: "Run ClickBench on ClickHouse Cloud"
 on:
   workflow_dispatch:  # This allows manual trigger from the UI
+    inputs:
+      max_parallel:
+        description: "Max concurrent services (0 = no cap)"
+        required: false
+        default: "0"
   schedule:
     - cron: '10 10 * * *'
 
@@ -13,6 +18,7 @@ jobs:
     env:
       CI_COMMIT_MESSAGE: "[bot] update results for ClickHouse Cloud"
       CI_COMMIT_AUTHOR: github
+      MAX_PARALLEL: ${{ github.event.inputs.max_parallel || '0' }}
     steps:
     - uses: actions/checkout@v3
     - run: |
@@ -24,21 +30,52 @@ jobs:
           echo "Required secrets are not set. Skipping workflow."
           exit 0
         fi
-
         cd clickhouse-cloud
         curl https://clickhouse.com/ | sh
         sudo ./clickhouse install -y
+    - name: Install diagnostics tools
+      run: |
+        sudo apt-get update -y
+        sudo DEBIAN_FRONTEND=noninteractive apt-get install -y sysstat iproute2 traceroute moreutils procps dnsutils
+    - name: Start diagnostics collectors
+      working-directory: clickhouse-cloud
+      run: |
+        mkdir -p diag
+        # CPU usage per core every second
+        (mpstat -P ALL 1 > diag/mpstat.log 2>&1 &)
+        # System runq/mem/io snapshot every second
+        (vmstat 1 > diag/vmstat.log 2>&1 &)
+        # Network device and TCP stats every second (ignore if sar not available)
+        (sar -n DEV 1 > diag/sar_net_dev.log 2>&1 &) || true
+        (sar -n TCP,ETCP 1 > diag/sar_tcp.log 2>&1 &) || true
+        # Socket summary snapshot once per second
+        (for i in $(seq 1 3600); do date +%FT%T%z >> diag/ss.log; ss -s >> diag/ss.log; sleep 1; done) &
         
         bash combinations.sh
         bash collect-results.sh
         
+    - name: Upload diagnostics and raw run outputs
+      uses: actions/upload-artifact@v4
+      with:
+        name: clickhouse-cloud-diag-${{ github.run_id }}
+        path: |
+          clickhouse-cloud/diag/**
+          clickhouse-cloud/csp-*/netdebug.txt
+          clickhouse-cloud/csp-*/chclient_errors.log
+          clickhouse-cloud/csp-*/result
+          clickhouse-cloud/csp-*/state
+          clickhouse-cloud/results/*.json
+        if-no-files-found: ignore
+    - name: Commit results
+      working-directory: clickhouse-cloud
+      run: |
         git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}"
         git config --global user.email "${{ env.CI_COMMIT_AUTHOR }}@users.noreply.github.com"
-        
         git pull
-        git add results/*.json
-        if git status | grep -q modified
-        then
+        git add results/*.json || true
+        if git status --porcelain | grep -qE "^ M|^A |^\?\? "; then
           git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
           git push
+        else
+          echo "No changes to commit."
         fi

clickhouse-cloud/cloud-api.sh

@@ -86,9 +86,27 @@ done
 
 echo "Waiting for clickhouse-server to start"
 
+# Collect quick network/TLS diagnostics for this run
+{
+    echo "==== NET DEBUG $(date -Iseconds) ===="
+    echo "Host: $(hostname)"
+    echo "Kernel: $(uname -a)"
+    echo "ulimit -n: $(ulimit -n 2>/dev/null || echo n/a)"
+    echo "FQDN: ${FQDN}"
+    echo "IP route:"; ip route || true
+    echo "IP addr:"; ip -brief address || ip addr || true
+    echo "DNS resolution for ${FQDN}:"; getent hosts "${FQDN}" || nslookup "${FQDN}" 2>&1 || host "${FQDN}" 2>&1 || true
+    echo "Socket summary:"; ss -s || true
+    echo "Public IP (best-effort):"; curl -fsS --max-time 3 https://ifconfig.me || echo "n/a"
+    echo "OpenSSL s_client (short):"; timeout 10s openssl s_client -connect "${FQDN}:9440" -servername "${FQDN}" -brief </dev/null 2>&1 || true
+    echo "Traceroute TCP:9440:"; timeout 20s traceroute -T -p 9440 -n "${FQDN}" 2>&1 || traceroute -n "${FQDN}" 2>&1 || true
+} > "${TMPDIR}/netdebug.txt" 2>&1
+
 for i in {1..1000}
 do
-    clickhouse-client --host "$FQDN" --password "$PASSWORD" --secure --query "SELECT 1" && break
+    if clickhouse-client --host "$FQDN" --password "$PASSWORD" --secure --query "SELECT 1" 1>/dev/null 2>>"${TMPDIR}/chclient_errors.log"; then
+        break
+    fi
     sleep 1
     if [[ $i == 1000 ]]
     then

clickhouse-cloud/combinations.sh

@@ -8,12 +8,22 @@ PROVIDER=aws
 REGION='us-east-1'
 PARALLEL_REPLICA=false
 
+# Optional cap on parallel jobs; set MAX_PARALLEL=N in the environment to enable.
+throttle() {
+    local max=${MAX_PARALLEL:-0}
+    if [[ -n "$max" && "$max" =~ ^[0-9]+$ && $max -gt 0 ]]; then
+        while [[ $(jobs -rp | wc -l) -ge $max ]]; do
+            sleep 1
+        done
+    fi
+}
+
 for REPLICAS in 1
 do
     for MEMORY in 8 12
     do
         export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
-        ./cloud-api.sh &
+    throttle; ./cloud-api.sh &
         sleep 10 # Prevent "Too many requests" to the API
     done
 done
@@ -23,7 +33,7 @@ do
     for MEMORY in 8 12 16 32 64 120 236
     do
         export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
-        ./cloud-api.sh &
+    throttle; ./cloud-api.sh &
         sleep 10
     done
 done
@@ -36,7 +46,7 @@ do
     for MEMORY in 8 12
     do
         export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
-        ./cloud-api.sh &
+    throttle; ./cloud-api.sh &
         sleep 10
     done
 done
@@ -46,7 +56,7 @@ do
     for MEMORY in 8 12 16 32 64 120 236
     do
         export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
-        ./cloud-api.sh &
+    throttle; ./cloud-api.sh &
         sleep 10
     done
 done
@@ -59,7 +69,7 @@ do
     for MEMORY in 8 12
     do
         export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
-        ./cloud-api.sh &
+    throttle; ./cloud-api.sh &
         sleep 10
     done
 done
@@ -69,7 +79,7 @@ do
     for MEMORY in 8 12 16 32 64 120
     do
         export PROVIDER REPLICAS REGION MEMORY PARALLEL_REPLICA
-        ./cloud-api.sh &
+    throttle; ./cloud-api.sh &
         sleep 10
     done
 done