diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 8e79d73ea..20c6ef13f 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -34,11 +34,10 @@ on: required: false default: true push-dirty: - description: Push scanned images that have vulnerabilities? + description: Push scanned images that have critical vulnerabilities? type: boolean required: false - # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures - default: true + default: false env: ANSIBLE_FORCE_COLOR: True @@ -180,7 +179,7 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - name: Create build logs output directory - run: mkdir image-build-logs + run: mkdir image-build-logs - name: Build kolla overcloud images id: build_overcloud_images @@ -239,9 +238,16 @@ jobs: run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt if: inputs.push + # NOTE(seunghun1ee): This always appends dirty images with CVEs severity lower than critical. + # This should be reverted when it's decided to filter high level CVEs as well. - name: Append dirty images to push list run: | cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt + if: ${{ inputs.push }} + + - name: Append images with critical vulnerabilities to push list + run: | + cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt if: ${{ inputs.push && inputs.push-dirty }} - name: Push images @@ -253,7 +259,7 @@ jobs: while read -r image; do # Retries! - for i in {1..5}; do + for i in {1..5}; do if docker push $image; then echo "Pushed $image" break @@ -287,8 +293,15 @@ jobs: run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi if: ${{ !cancelled() }} - - name: Fail when images failed scanning - run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi + # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected. + # This can be used again instead of "Fail when critical vulnerabilities are found" when it's + # decided to fail the job on detecting high CVEs as well. + # - name: Fail when images failed scanning + # run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi + # if: ${{ !inputs.push-dirty && !cancelled() }} + + - name: Fail when critical vulnerabilities are found + run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi if: ${{ !inputs.push-dirty && !cancelled() }} # NOTE(mgoddard): Trigger another CI workflow in the diff --git a/doc/source/operations/octavia.rst b/doc/source/operations/octavia.rst index e2ab74c4a..f884d130f 100644 --- a/doc/source/operations/octavia.rst +++ b/doc/source/operations/octavia.rst @@ -2,6 +2,8 @@ Octavia ======= +.. _Amphora image: + Building and rotating amphora images ==================================== diff --git a/doc/source/operations/upgrading.rst b/doc/source/operations/upgrading.rst index 5fce4deee..cbb79e455 100644 --- a/doc/source/operations/upgrading.rst +++ b/doc/source/operations/upgrading.rst @@ -458,8 +458,8 @@ To upgrade the Ansible control host: Syncing Release Train artifacts ------------------------------- -New `StackHPC Release Train <../configuration/release-train>` content should be -synced to the local Pulp server. This includes host packages (Deb/RPM) and +New `StackHPC Release Train <../configuration/release-train>`_ content should +be synced to the local Pulp server. This includes host packages (Deb/RPM) and container images. .. _sync-rt-package-repos: @@ -1067,6 +1067,12 @@ scope of the upgrade: kayobe overcloud service upgrade --tags config --kolla-tags keystone +Updating the Octavia Amphora Image +---------------------------------- + +If using Octavia with the Amphora driver, you should :ref:`build a new amphora +image `. + Testing ------- diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules index 4b5bd01f2..f14120b8f 100644 --- a/etc/kayobe/kolla/config/prometheus/system.rules +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -25,7 +25,7 @@ groups: description: "Available memory is {{ $value }} GiB." - alert: LowSwapSpace - expr: (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_warning_threshold_ratio }}{% raw %} + expr: node_memory_SwapTotal_bytes > 0 and (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_warning_threshold_ratio }}{% raw %} for: 1m labels: severity: warning @@ -34,7 +34,7 @@ groups: description: "Available swap space is {{ $value | humanizePercentage }}. Running out of swap space causes OOM Kills." - alert: LowSwapSpace - expr: (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_critical_threshold_ratio }}{% raw %} + expr: node_memory_SwapTotal_bytes > 0 and (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_critical_threshold_ratio }}{% raw %} for: 1m labels: severity: critical diff --git a/tools/scan-images.sh b/tools/scan-images.sh index b8cde6ede..3ee7d3edf 100755 --- a/tools/scan-images.sh +++ b/tools/scan-images.sh @@ -27,11 +27,13 @@ docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2" > $1-sca images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:) # Ensure output files exist -touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt +touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt # If Trivy detects no vulnerabilities, add the image name to clean-images.txt. # If there are vulnerabilities detected, add it to dirty-images.txt and # generate a csv summary +# If the image contains at least one critical vulnerabilities, add it to +# critical-images.txt for image in $images; do filename=$(basename $image | sed 's/:/\./g') imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g') @@ -59,15 +61,13 @@ for image in $images; do # Add the image to the clean list echo "${image}" >> image-scan-output/clean-images.txt else - # Add the image to the dirty list - echo "${image}" >> image-scan-output/dirty-images.txt - + # Write a header for the summary CSV echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv # Write the summary CSV data - jq -r '.Results[] - | select(.Vulnerabilities) + jq -r '.Results[] + | select(.Vulnerabilities) | .Vulnerabilities # Ignore packages with "kernel" in the PkgName | map(select(.PkgName | test("kernel") | not )) @@ -82,9 +82,17 @@ for image in $images; do .[0].PrimaryURL, .[0].Severity ] - ) - | .[] + ) + | .[] | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv + + if [ $(grep "CRITICAL" image-scan-output/${filename}.summary.csv -c) -gt 0 ]; then + # If the image contains critical vulnerabilities, add the image to critical list + echo "${image}" >> image-scan-output/critical-images.txt + else + # Otherwise, add the image to the dirty list + echo "${image}" >> image-scan-output/dirty-images.txt + fi fi rm .trivyignore done