Merge branch 'stackhpc/yoga' into yoga-infiniband

Author: tomclark0

.automation.conf/config.sh

@@ -24,6 +24,7 @@ if [ ! -z ${KAYOBE_ENVIRONMENT:+x} ]; then
     # SMSLab is currently running with 1G switches. This causes tests using volumes and images to fail if
     # the concurrency is set too high.
     export TEMPEST_CONCURRENCY=1
+    export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST="ci-multinode-platform.2022.11"
     # Uncomment this to perform a full tempest test
     # export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full
     # export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode-tempest-full

.automation.conf/tempest/skip-lists/ci-multinode-platform.2022.11

@@ -0,0 +1,2 @@
+tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_pagination: "Fails without public TLS"
+tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_pagination: "Fails without public TLS"

.automation.conf/tempest/skip-lists/ci-multinode-tempest-full

@@ -1 +1,3 @@
+tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_pagination: "Fails without public TLS"
+tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_pagination: "Fails without public TLS"
 tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_subnet_details.*: "Cirros image doesn't have '/var/run/udhcpc.eth0.pid"

.github/workflows/stackhpc-all-in-one.yml

@@ -297,7 +297,7 @@ jobs:
             -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
-            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack -e rally_no_sensitive_log=false
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 

.github/workflows/stackhpc-container-image-build.yml

@@ -39,11 +39,10 @@ on:
         required: false
         default: true
       push-dirty:
-        description: Push scanned images that have vulnerabilities?
+        description: Push scanned images that have critical vulnerabilities?
         type: boolean
         required: false
-        # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures
-        default: true
+        default: false
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -181,7 +180,7 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Create build logs output directory
-        run: mkdir image-build-logs 
+        run: mkdir image-build-logs
 
       - name: Build kolla overcloud images
         id: build_overcloud_images
@@ -233,16 +232,23 @@ jobs:
         run: mv image-scan-output image-build-logs/image-scan-output
 
       - name: Fail if no images have passed scanning
-        run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
+        run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi
         if: ${{ !inputs.push-dirty }}
 
       - name: Copy clean images to push-attempt-images list
         run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
         if: inputs.push
 
+      # NOTE(seunghun1ee): This always appends dirty images with CVEs severity lower than critical.
+      # This should be reverted when it's decided to filter high level CVEs as well.
       - name: Append dirty images to push list
         run: |
           cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
+        if: ${{ inputs.push }}
+
+      - name: Append images with critical vulnerabilities to push list
+        run: |
+          cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt
         if: ${{ inputs.push && inputs.push-dirty }}
 
       - name: Push images
@@ -254,7 +260,7 @@ jobs:
 
           while read -r image; do
             # Retries!
-            for i in {1..5}; do 
+            for i in {1..5}; do
               if docker push $image; then
                 echo "Pushed $image"
                 break
@@ -288,8 +294,15 @@ jobs:
         run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
         if: ${{ !cancelled() }}
 
-      - name: Fail when images failed scanning
-        run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi
+      # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected.
+      # This can be used again instead of "Fail when critical vulnerabilities are found" when it's
+      # decided to fail the job on detecting high CVEs as well.
+      # - name: Fail when images failed scanning
+      #   run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi
+      #   if: ${{ !inputs.push-dirty && !cancelled() }}
+
+      - name: Fail when critical vulnerabilities are found
+        run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
         if: ${{ !inputs.push-dirty && !cancelled() }}
 
       # NOTE(mgoddard): Trigger another CI workflow in the

etc/kayobe/ansible/cis.yml

@@ -18,6 +18,17 @@
         state: absent
       when: ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '8'
 
+    - name: Ensure service accounts have no expiry options set
+      # This is to workaround an issue where we set the expiry to 365 days on kayobe
+      # service accounts in a previous iteration of the CIS benchmark hardening
+      # defaults. This should restore the defaults and can eventually be removed.
+      command: chage -m 0 -M 99999 -W 7 -I -1 {{ item }}
+      become: true
+      changed_when: false
+      with_items:
+        - "{{ kayobe_ansible_user }}"
+        - "{{ kolla_ansible_user }}"
+
     - include_role:
         name: ansible-lockdown.rhel8_cis
       when: ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '8'

etc/kayobe/ansible/requirements.yml

@@ -9,7 +9,7 @@ collections:
   - name: stackhpc.pulp
     version: 0.4.1
   - name: stackhpc.hashicorp
-    version: 2.4.0
+    version: 2.5.0
   - name: stackhpc.kayobe_workflows
     version: 1.0.3
 roles:

etc/kayobe/ansible/ubuntu-upgrade.yml

@@ -5,6 +5,7 @@
   hosts: overcloud:infra-vms:seed:seed-hypervisor
   vars:
     ansible_python_interpreter: /usr/bin/python3
+    reboot_timeout_s: "{{ 20 * 60 }}"
   tasks:
     - name: Assert that hosts are running Ubuntu Focal
       assert:
@@ -37,7 +38,7 @@
 
     - name: Reboot to apply updates
       reboot:
-        reboot_timeout: 1200
+        reboot_timeout: "{{ reboot_timeout_s }}"
         connect_timeout: 600
       become: true
       when: file_status.stat.exists
@@ -81,16 +82,24 @@
   hosts: overcloud:infra-vms:seed:seed-hypervisor
   vars:
     ansible_python_interpreter: /usr/bin/python3
+    reboot_timeout_s: "{{ 20 * 60 }}"
   tasks:
     - name: Ensure Jammy repo definitions do not exist in sources.list
       blockinfile:
         path: /etc/apt/sources.list
         state: absent
       become: true
 
+    - name: Ensure Kolla Ansible Docker repo definition does not exist
+      file:
+        path: /etc/apt/sources.list.d/docker.list
+        state: absent
+      become: true
+      when: apt_repositories | selectattr('url', 'match', '.*docker-ce.*') | list | length > 0
+
     - name: Reboot and wait
       reboot:
-        reboot_timeout: 1200
+        reboot_timeout: "{{ reboot_timeout_s }}"
         connect_timeout: 600
       become: true
 

etc/kayobe/apt.yml

@@ -52,20 +52,29 @@ stackhpc_apt_repositories:
     suites: "{{ ansible_facts.distribution_release }} {{ ansible_facts.distribution_release }}-updates {{ ansible_facts.distribution_release }}-backports"
     components: main restricted universe multiverse
     architecture: amd64
+    required: true
   - url: "{{ stackhpc_repo_ubuntu_focal_security_url if ansible_facts.distribution_release == 'focal' else stackhpc_repo_ubuntu_jammy_security_url }}"
     suites: "{{ ansible_facts.distribution_release }}-security"
     components: main restricted universe multiverse
     architecture: amd64
-  - url: "{{ stackhpc_repo_docker_ce_ubuntu_url }}"
-    suites: "{{ ansible_facts.distribution_release }}"
+    required: true
+  - url: "{{ stackhpc_repo_ubuntu_jammy_cve_2024_6387_url }}"
+    suites: "pulp"
+    components: upload
+    architecture: amd64
+    trusted: yes
+    required: "{{ ansible_facts.distribution_release == 'jammy' }}"
+  - url: "{{ stackhpc_repo_docker_ce_ubuntu_focal_url if ansible_facts.distribution_release == 'focal' else stackhpc_repo_docker_ce_ubuntu_jammy_url }}"
+    suites: "{{ ansible_facts.distribution_release  }}"
     components: stable
     signed_by: docker.asc
     architecture: amd64
+    required: true
 
 # Do not replace apt configuration for non-overcloud hosts. This can result in
 # errors if apt reconfiguration is performed before local repository mirrors
 # are deployed.
-apt_repositories: "{{ stackhpc_apt_repositories if 'overcloud' in group_names else [] }}"
+apt_repositories: "{{ stackhpc_apt_repositories | selectattr('required') | list if 'overcloud' in group_names else [] }}"
 
 # Whether to disable repositories in /etc/apt/sources.list. This may be used
 # when replacing the distribution repositories via apt_repositories.

etc/kayobe/dnf.yml

@@ -215,6 +215,15 @@ dnf_custom_repos_rocky_9:
     gpgcheck: yes
     username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
     password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
+  security-common:
+    baseurl: "{{ stackhpc_repo_rocky_9_sig_security_common_url }}"
+    description: "Rocky Linux $releasever - SIG Security Common"
+    file: Rocky-SIG-Security-Common
+    gpgkey: "{{ rocky_9_sig_security_gpg_key }}"
+    gpgcheck: yes
+    includepkgs: "openssh*"
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # Whether to enable EPEL repositories. This affects RedHat-based systems only.
 dnf_enable_epel: "{{ dnf_install_epel | bool }}"
@@ -227,6 +236,7 @@ dnf_epel_8_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-
 dnf_epel_9_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9"
 
 rocky_9_gpg_key: "https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9"
+rocky_9_sig_security_gpg_key: "https://dl.rockylinux.org/pub/sig/9/security/x86_64/security-common/RPM-GPG-KEY-Rocky-SIG-Security"
 
 # Whether to install the epel-release package. This affects RedHat-based
 # systems only. Default value is 'false'.