[WIP]: Revamp CI reporting

.github/workflows/job-test-in-container.yml

@@ -1,4 +1,4 @@
 # This job runs integration tests inside a container, for all supported variants (ipv6, canary, etc)
 # Note that it is linux and nerdctl (+/- gomodjail) only.
 name: job-test-in-container
 
@@ -35,6 +35,10 @@
         required: false
         default: false
         type: boolean
+    outputs:
+      artifact:
+        description: "Artifact generated by this job"
+        value: ${{ jobs.test.outputs.artifact }}
 
 env:
   GOTOOLCHAIN: local
@@ -55,6 +59,8 @@
     defaults:
       run:
         shell: bash
+    outputs:
+      artifact: ${{ steps.artifact-upload.outputs.artifact-url }}
 
     env:
       # https://github.com/containerd/nerdctl/issues/622
@@ -161,9 +167,9 @@
             && args=(test-integration ./hack/test-integration.sh -test.allow-modify-users=true) \
             || args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
           if [ "${{ inputs.ipv6 }}" == true ]; then
-            docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${{ inputs.binary }}
+            docker run --name test-runner --network host -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${{ inputs.binary }}
           else
-            docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.target=${{ inputs.binary }}
+            docker run --name test-runner -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.target=${{ inputs.binary }}
           fi
       # FIXME: this NEEDS to go away
       - name: "Run: integration tests (flaky)"
@@ -175,7 +181,42 @@
             && args=(test-integration ./hack/test-integration.sh) \
             || args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
           if [ "${{ inputs.ipv6 }}" == true ]; then
-            docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${{ inputs.binary }}
+            docker run --name test-runner-flaky --network host -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${{ inputs.binary }}
           else
-            docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.target=${{ inputs.binary }}
+            docker run --name test-runner-flaky -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.target=${{ inputs.binary }}
           fi
+
+      - name: "Wrap: collector"
+        if: ${{ failure() || success() }}
+        run: |
+          # Get the reports from inside the containers
+          [ "${{ inputs.target }}" == "rootful" ] && src=/root || src=/home/rootless
+          mkdir -p ~/report
+          docker cp test-runner:$src/nerdctl-test-report ~/report/main || true
+          # Flaky may not have run
+          docker cp test-runner-flaky:$src/nerdctl-test-report ~/report/flaky 2>/dev/null || true
+          # Add metadata info to the runs
+          . ./mod/wax/scripts/collector.sh
+          collect::metadata \
+            "runner=${{ inputs.runner }}" \
+            "binary=${{ inputs.binary }}" \
+            "canary=${{ inputs.canary }}" \
+            "target=${{ inputs.target }}" \
+            "ipv6=${{ inputs.ipv6 }}" \
+            "containerd-version=${{ inputs.containerd-version }}" \
+            "rootlesskit-version=${{ inputs.rootlesskit-version }}" \
+            "attempt=$GITHUB_RUN_ATTEMPT" \
+            "sha=$GITHUB_SHA" \
+            "id=$GITHUB_RUN_ID" \
+            "number=$GITHUB_RUN_NUMBER" \
+            > ~/report/main/metadata.json || true
+            [ ! -e ~/report/flaky ] || cp ~/report/main/metadata.json ~/report/flaky/metadata.json || true
+
+      - name: "Wrap: upload"
+        id: artifact-upload
+        if: ${{ failure() || success() }}
+        uses: actions/upload-artifact@v4
+        with:
+          path: ~/report/*
+          retention-days: 1
+          name: wax-${{ inputs.runner }}-${{ inputs.binary }}-${{ inputs.canary }}-${{ inputs.target }}-${{ inputs.ipv6 }}-${{ inputs.containerd-version }}-${{ inputs.rootlesskit-version }}

.github/workflows/job-test-in-host.yml

@@ -43,6 +43,10 @@ on:
       linux-cni-sha:
         required: true
         type: string
+    outputs:
+      artifact:
+        description: "Artifact generated by this job"
+        value: ${{ jobs.test.outputs.artifact }}
 
 env:
   GOTOOLCHAIN: local
@@ -60,6 +64,8 @@ jobs:
     defaults:
       run:
         shell: bash
+    outputs:
+      artifact: ${{ steps.artifact-upload.outputs.artifact-url }}
 
     env:
       SHOULD_RUN: "yes"
@@ -107,6 +113,7 @@ jobs:
           if [ "${{ contains(inputs.binary, 'docker') }}" == true ]; then
             echo "::group:: configure cdi for docker"
             sudo mkdir -p /etc/docker
+            sudo touch /etc/docker/daemon.json
             sudo jq '.features.cdi = true' /etc/docker/daemon.json | sudo tee /etc/docker/daemon.json.tmp && sudo mv /etc/docker/daemon.json.tmp /etc/docker/daemon.json
             echo "::endgroup::"
             echo "::group:: downgrade docker to the specific version we want to test (${{ inputs.docker-version }})"
@@ -183,22 +190,30 @@ jobs:
           make install-dev-tools
           echo "::endgroup::"
 
+
+      - if: ${{ env.SHOULD_RUN == 'yes' }}
+        name: "Init: prepare artifacts directories"
+        run: |
+          mkdir -p ~/report/main
+          mkdir -p ~/report/ipv6
+          mkdir -p ~/report/flaky
+
       # ipv6 is tested only on linux
-      - if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }}
+      - if: ${{ env.SHOULD_RUN == 'yes' && contains(inputs.runner, 'ubuntu' )}}
         name: "Run (linux): integration tests (IPv6)"
         run: |
           . ./hack/github/action-helpers.sh
           github::md::h2 "ipv6" >> "$GITHUB_STEP_SUMMARY"
 
-          ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-ipv6
+          WAX_REPORT_LOCATION=$HOME/report/ipv6 ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-ipv6
 
       - if: ${{ env.SHOULD_RUN == 'yes' }}
         name: "Run: integration tests"
         run: |
           . ./hack/github/action-helpers.sh
           github::md::h2 "non-flaky" >> "$GITHUB_STEP_SUMMARY"
 
-          ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=false
+          WAX_REPORT_LOCATION=$HOME/report/main ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=false
 
       # FIXME: this must go
       - if: ${{ env.SHOULD_RUN == 'yes' }}
@@ -207,4 +222,42 @@ jobs:
           . ./hack/github/action-helpers.sh
           github::md::h2 "flaky" >> "$GITHUB_STEP_SUMMARY"
 
-          ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=true
+          WAX_REPORT_LOCATION=$HOME/report/flaky ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=true
+
+      - name: "Wrap: collector"
+        if: ${{ env.SHOULD_RUN == 'yes' && (failure() || success()) }}
+        run: |
+          # Add metadata info to the runs
+          . ./mod/wax/scripts/collector.sh
+          collect::metadata \
+            "runner=${{ inputs.runner }}" \
+            "binary=${{ inputs.binary }}" \
+            "canary=${{ inputs.canary }}" \
+            "ipv6=false" \
+            "attempt=$GITHUB_RUN_ATTEMPT" \
+            "sha=$GITHUB_SHA" \
+            "id=$GITHUB_RUN_ID" \
+            "number=$GITHUB_RUN_NUMBER" \
+            > ~/report/main/metadata.json || true
+
+          collect::metadata \
+            "runner=${{ inputs.runner }}" \
+            "binary=${{ inputs.binary }}" \
+            "canary=${{ inputs.canary }}" \
+            "ipv6=true" \
+            "attempt=$GITHUB_RUN_ATTEMPT" \
+            "sha=$GITHUB_SHA" \
+            "id=$GITHUB_RUN_ID" \
+            "number=$GITHUB_RUN_NUMBER" \
+            > ~/report/ipv6/metadata.json || true
+
+          cp ~/report/main/metadata.json ~/report/flaky/metadata.json || true
+
+      - name: "Wrap: upload"
+        id: artifact-upload
+        if: ${{ env.SHOULD_RUN == 'yes' && (failure() || success()) }}
+        uses: actions/upload-artifact@v4
+        with:
+          path: ~/report/*
+          retention-days: 1
+          name: wax-${{ inputs.runner }}-${{ inputs.binary }}-${{ inputs.canary }}

.github/workflows/workflow-lint.yml

@@ -78,3 +78,13 @@ jobs:
       go-version: ${{ matrix.go-version }}
       runner: ubuntu-24.04
       canary: ${{ matrix.canary && true || false }}
+
+
+  reporter:
+    if: ${{ failure() || success() }}
+    name: "DEBUG"
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Process
+        run: |
+          printf '::error title=TESTTHIS,file=.github/workflows/job-test-in-container.yml,line=1::ONE<br><br>TWO %0A; %0A; %0A; `THREE`\n'

.github/workflows/workflow-test.yml

@@ -147,3 +147,33 @@ jobs:
       containerd-service-sha: 1941362cbaa89dd591b99c32b050d82c583d3cd2e5fa63085d7017457ec5fca8
       linux-cni-version: v1.7.1
       linux-cni-sha: 1a28a0506bfe5bcdc981caf1a49eeab7e72da8321f1119b7be85f22621013098
+
+  reporter:
+    if: ${{ failure() || success() }}
+    name: "reporter${{ inputs.hack }}"
+    needs:
+      - test-integration-host
+      - test-integration-container
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Fetch Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: "Init: install go"
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5  # v5.5.0
+        with:
+          go-version: 1.24
+          check-latest: true
+      - name: Download all workflow run artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: ~/report
+      - name: Process
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          cd ./mod/wax
+          go build -o ../../wax ./cmd/wax
+          cd -
+          ./wax ~/report

cmd/nerdctl/main_test.go

@@ -33,6 +33,11 @@
 	testutil.M(m)
 }
 
+func TestWax(t *testing.T) {
+	t.Log("This test is voluntarily failing")
+	t.FailNow()
+}
+
 // TestUnknownCommand tests https://github.com/containerd/nerdctl/issues/487
 func TestUnknownCommand(t *testing.T) {
 	testCase := nerdtest.Setup()

hack/github/action-helpers.sh

@@ -117,3 +117,8 @@ github::timer::format() {
   [[ "$m" == 0 ]] || printf "%d minutes " "$m"
   printf '%d seconds' "$s"
 }
+
+#          echo "::error title=ErrorReport::MEH .github/workflows/job-test-in-host.yml${{steps.artifact-upload.outputs.artifact-url}}"
+#          echo "::notice title=NoticeReport::SHEESH ${{steps.artifact-upload.outputs.artifact-url}}"
+#          echo "::error file=cmd/nerdctl/main_test_test.go,line=1,endLine=10,title=AgainErrorReport::FOO ${{steps.artifact-upload.outputs.artifact-url}}"
+#          echo "::error file=cmd/nerdctl/main_test.go,line=38,endLine=41,title=AgainErrorReport::BLA ${{steps.artifact-upload.outputs.artifact-url}}"

hack/github/gotestsum-reporter.sh

@@ -14,6 +14,11 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+# DEPRECATED: favor post-processing inside integration.sh rather than using a post-run hook.
+# Reasons are:
+# - post-run hook do not work on windows
+# - post-run hook is limited to a single run of gotestsum, while it is desirable to process multiple runs together
+
 # shellcheck disable=SC2034,SC2015
 set -o errexit -o errtrace -o functrace -o nounset -o pipefail
 root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"

hack/test-integration.sh

@@ -19,38 +19,79 @@ set -o errexit -o errtrace -o functrace -o nounset -o pipefail
 root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"
 readonly root
 
-if [[ "$(id -u)" = "0" ]]; then
-  # Ensure securityfs is mounted for apparmor to work
-  if ! mountpoint -q /sys/kernel/security; then
-    mount -tsecurityfs securityfs /sys/kernel/security
-  fi
+# If no argument is provided, run both flaky and not-flaky test suites.
+if [ "$#" == 0 ]; then
+  "$root"/integration.sh -test.only-flaky=false
+  "$root"/integration.sh -test.only-flaky=true
+  exit
 fi
 
+##### Import helper libraries
+# shellcheck source=/dev/null
+. "$root"/../mod/wax/scripts/collector.sh
+
+##### Configuration
+# Where to store report files
+readonly report_location="${WAX_REPORT_LOCATION:-$HOME/nerdctl-test-report}"
+# Where to store gotestsum log file
+readonly gotestsum_log_main="$report_location"/test-integration.log
+readonly gotestsum_log_flaky="$report_location"/test-integration-flaky.log
+# Total run timeout
 readonly timeout="60m"
+# Number of retries for flaky tests
 readonly retries="2"
-readonly needsudo="${WITH_SUDO:-}"
-
-# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
-args=(--format=testname --jsonfile /tmp/test-integration.log --packages="$root"/../cmd/nerdctl/...)
-# FIXME: not working on windows. Need to change approach: move away from --post-run-command and
-# just process the log file. This might also allow multi-steps/multi-target results aggregation.
-[ "$(uname -s)" != "Linux" ] || args+=(--post-run-command "$root"/github/gotestsum-reporter.sh)
-
-if [ "$#" == 0 ]; then
-  "$root"/test-integration.sh -test.only-flaky=false
-  "$root"/test-integration.sh -test.only-flaky=true
-  exit
-fi
+readonly need_sudo="${WITH_SUDO:-}"
 
+##### Prepare gotestsum arguments
+mkdir -p "$report_location"
+# Format and packages to test
+args=(--format=testname --packages="$root"/../cmd/nerdctl/...)
+# Log file
+gotestsum_log="$gotestsum_log_main"
 for arg in "$@"; do
   if [ "$arg" == "-test.only-flaky=true" ] || [ "$arg" == "-test.only-flaky" ]; then
     args+=("--rerun-fails=$retries")
+    gotestsum_log="$gotestsum_log_flaky"
     break
   fi
 done
+args+=(--jsonfile "$gotestsum_log" --)
+
+##### Append go test arguments
+# Honor sudo
+[ "$need_sudo" != true ] && [ "$need_sudo" != yes ] && [ "$need_sudo" != 1 ] || args+=(-exec sudo)
+# About `-p 1`, see https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
+args+=(-timeout="$timeout" -p 1 -args -test.allow-kill-daemon "$@")
 
-if [ "$needsudo" == "true" ] || [ "$needsudo" == "yes" ] || [ "$needsudo" == "1" ]; then
-  gotestsum "${args[@]}" -- -timeout="$timeout" -p 1 -exec sudo -args -test.allow-kill-daemon "$@"
-else
-  gotestsum "${args[@]}" -- -timeout="$timeout" -p 1 -args -test.allow-kill-daemon "$@"
+# FIXME: this should not be the responsibility of the test script
+# Instead, it should be in the Dockerfile (or other stack provisioning script) - eg: /etc/systemd/system/securityfs.service
+# [Unit]
+# Description=Kernel Security File System
+# DefaultDependencies=no
+# Before=sysinit.target
+# Before=apparmor.service
+# ConditionSecurity=apparmor
+# ConditionPathIsMountPoint=!/sys/kernel/security
+#
+# [Service]
+# Type=oneshot
+# ExecStart=/bin/mount -t securityfs -o nosuid,nodev,noexec securityfs /sys/kernel/security
+#
+# [Install]
+# WantedBy=sysinit.target
+if [[ "$(id -u)" = "0" ]]; then
+  # Ensure securityfs is mounted for apparmor to work
+  if ! mountpoint -q /sys/kernel/security; then
+    mount -tsecurityfs securityfs /sys/kernel/security
+  fi
 fi
+
+##### Run it
+ex=0
+gotestsum "${args[@]}" || ex=$?
+
+##### Post: collect logs into the report location
+collect::logs "$report_location"
+
+# Honor gotestsum exit code
+exit "$ex"