Merge branch 'main' into feat/auto-gres

Author: sjpb

.ansible-lint.yml

@@ -6,8 +6,6 @@ skip_list:
   - jinja[spacing]
   - galaxy[no-changelog]
   - meta-runtime[unsupported-version]
-
-warn_list:
   - name[missing]
   - name[play]
   - var-naming

.github/workflows/stackhpc.yml

@@ -107,7 +107,12 @@ jobs:
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd "$STACKHPC_TF_DIR"
-          tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
+          max_retries=3
+          delay=30
+          for i in $(seq 1 $max_retries); do
+            tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars" && break
+            [ "$i" -lt "$max_retries" ] && sleep $delay || exit 1
+          done
 
       - name: Delete infrastructure if provisioning failed
         run: |

ansible/adhoc/sync-pulp.yml

@@ -5,7 +5,5 @@
         name: pulp_site
         tasks_from: sync.yml
       vars:
-        pulp_site_target_arch: "x86_64"
-        pulp_site_target_distribution: "rocky"
         # default distribution to *latest* specified for baseos repo:
         pulp_site_target_distribution_version: "{{ dnf_repos_repos['baseos'].keys() | map('float') | sort | last }}"

ansible/fatimage.yml

@@ -206,6 +206,7 @@
       ansible.builtin.include_role:
         name: hpctests
         tasks_from: source-hpl.yml
+      when: "'hpctests' in group_names"
 
 - hosts: prometheus
   become: true

ansible/roles/eessi/defaults/main.yaml

@@ -1,4 +1,6 @@
 ---
+cvmfs_release_version: "6-3"
+
 # Default to 10GB
 cvmfs_quota_limit_mb: 10000
 
@@ -9,4 +11,4 @@ cvmfs_config_default:
 cvmfs_config_overrides: {}
 cvmfs_config: "{{ cvmfs_config_default | combine(cvmfs_config_overrides) }}"
 
-cvmfs_gpg_checksum: "sha256:4ac81adff957565277cfa6a4a330cdc2ce5a8fdd73b8760d1a5a32bef71c4bd6"
+cvmfs_gpg_checksum: "sha256:5c60679d307a96524204c127250e8ebdda66a459659faa1718bdf32dde1d7069"

ansible/roles/eessi/tasks/install.yml

@@ -3,18 +3,17 @@
 - name: Download Cern GPG key
   # checkov:skip=CKV2_ANSIBLE_2: "Ensure that HTTPS url is used with get_url"
   ansible.builtin.get_url:
-    url: http://cvmrepo.web.cern.ch/cvmrepo/yum/RPM-GPG-KEY-CernVM
+    url: https://cvmrepo.web.cern.ch/cvmrepo/yum/RPM-GPG-KEY-CernVM-2048
     dest: ./cvmfs-key.gpg
     checksum: "{{ cvmfs_gpg_checksum }}"
     mode: "0644"
 
 - name: Import downloaded GPG key # noqa: no-changed-when
   ansible.builtin.command: rpm --import cvmfs-key.gpg # noqa: command-instead-of-module
+
 - name: Add CVMFS repo
-  # checkov:skip=CKV2_ANSIBLE_4: "Ensure that packages with untrusted or missing GPG signatures are not used by dnf"
   ansible.builtin.dnf:
-    name: https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm
-    disable_gpg_check: true
+    name: "https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-{{ cvmfs_release_version }}.noarch.rpm"
 
 - name: Install CVMFS
   ansible.builtin.dnf:

ansible/roles/nhc/README.md

@@ -37,6 +37,7 @@ When the `ansible/site.yml` playbook is run this will automatically:
 
    - Filesystem mounts
    - Ethernet interfaces
+   - InfiniBand interfaces
 
    See `/etc/nhc/nhc.conf` on a compute node for the full configuration.
 

ansible/roles/nhc/templates/nhc.conf.j2

@@ -8,13 +8,18 @@
 {# /efi is mounted both directly and via systemd1 autofs, which NHC can't cope with #}
 {# use `awk '{print $5 " " $10 " " $4 " " $9}' /proc/self/mountinfo | sort -k1` to check that is the only case #}
 {% set mount_mode = 'rw' if 'rw' in mount.options.split(',') else 'ro' %}
-{{ ansible_fqdn }} || check_fs_mount_{{ mount_mode }} -t "{{ mount.fstype }}" -s "{{ mount.device }}" -f "{{ mount.mount }}"
+{{ ansible_fqdn }} || check_fs_mount_{{ mount_mode }} -t "{{ mount.fstype }}" -s "{{ mount.device | replace(' ', '\\040') }}" -f "{{ mount.mount }}"
 {% endfor %}
 
 ## Ethernet interface checks
 {% for iface in ansible_interfaces | select('match', 'eth') %}
 {{ ansible_fqdn }} || check_hw_eth {{ iface }}
 {% endfor %}
 
+## InfiniBand interface checks
+{% for iface in ansible_interfaces | select('match', 'ib') %}
+{{ ansible_fqdn }} || check_hw_ib {{ (ansible_facts[iface]['speed'] / 1000) | int }} {{ iface }}
+{% endfor %}
+
 ## Site-specific checks
 {{ nhc_config_extra }}

dev/image-set-properties.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/bash
+# Set image properties correctly for Slurm Appliance images
+#
+# Usage:
+#   dev/image-set-properties.sh $IMAGE_NAME_OR_ID
+
+set -euo pipefail
+
+image=${1?param missing - image name or ID}
+echo getting image format ...
+format=$(openstack image show -c disk_format -f value "${image}")
+
+echo setting constant properties ...
+set -x
+openstack image set \
+--property hw_machine_type=q35 \
+--property hw_architecture=x86_64 \
+--property hw_vif_multiqueue_enabled=true \
+--property hw_firmware_type=uefi \
+--property os_distro=rocky \
+--property os_type=linux \
+--property os_admin_user=rocky \
+"$image"
+
+set +x
+if [[ "$format" = raw ]]; then
+    echo setting raw properties...
+    set -x
+    openstack image set \
+    --property hw_scsi_model=virtio-scsi \
+    --property hw_disk_bus=scsi \
+    "$image"
+else
+    echo setting qcow2 properties
+    set -x
+    openstack image set \
+    --property hw_scsi_model=virtio-scsi \
+    --property hw_disk_bus=scsi \
+    --property hw_scsi_model=virtio \
+    "$image"
+fi

docs/eessi.md

@@ -0,0 +1,97 @@
+# EESSI Guide
+
+## How to Load EESSI
+
+The EESSI environment can be initialise by running:
+
+```[bash]
+source /cvmfs/software.eessi.io/versions/2023.06/init/bash
+```
+
+This is non-reversible because it:
+
+- Changes your `$PATH`, `$MODULEPATH`, `$LD_LIBRARY_PATH`, and other critical environment variables.
+- Sets EESSI-specific variables such as `EESSI_ROOT`.
+
+However, it is the recommended method because it:
+
+- Detects system CPU architecture and OS.
+- Detects and configures GPU support.
+- Prepares the full EESSI software stack.
+- Sets up Lmod (environment module system).
+
+The [EESSI docs](https://www.eessi.io/docs/using_eessi/setting_up_environment/) offer another method to load EESSI. This alternative method only initialises the Lmod module system and does not load a platform-specific setup. For these reasons, it is recommended to use the method detailed above.
+
+Successful environment setup will show `{EESSI 2023.06}` at the start of your CLI.
+
+To deactivate your EESSI environment you can either restart your shell using `exec bash` or exit the shell by running `exit`.
+
+## GPU Support with EESSI
+
+To enable GPU support, the cluster must be running a site-specific image build that has CUDA enabled. For a guide on how to do this, please refer to [docs/image-build.md](../image-build.md).
+
+### Using GPUs
+
+All CUDA-enabled software in EESSI expects CUDA drivers in a specific `host_injections` directory.
+
+#### To expose the NVIDIA GPU drivers
+
+Use the `link_nvidia_host_libraries.sh` script, provided by EESSI, to symlink your GPU drivers into `host_injections`.
+
+```[bash]
+/cvmfs/software.eessi.io/versions/2023.06/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
+```
+
+Rerun this script when your NVIDIA GPU drivers are updated. It is also safe to rerun at any time as the script will detect if the driver versions have already been symlinked.
+
+### Building with GPUs
+
+Run `which nvcc` to confirm that the CUDA compiler is found.
+
+If `nvcc` is not found, add the CUDA path to your environment:
+
+```[bash]
+export PATH=/usr/local/cuda/bin:$PATH
+```
+
+`which nvcc` should now show the path to the CUDA compiler.
+
+#### Loading EESSI module for the GCC compiler
+
+Running `which gcc` with EESSI initialised should initially show a path `.../2023.06/compat...` which points to the compatibility compiler.
+
+It is important to load a `gcc` version that is compatible with the host's CUDA version. This can be found by referring to the table below:
+
+To find your cuda version run `nvcc --version`
+
+| **CUDA Version** | **Recommended GCC Module (EESSI 2023.06)**              |
+| ---------------- | ------------------------------------------------------- |
+| CUDA 13.0        | GCC/12.3.0                                              |
+| CUDA 12.3 – 12.5 | GCC/12.3.0 or GCC/12.2.0 (officially supports GCC 12.x) |
+| CUDA 12.0 – 12.2 | GCC/12.2.0 (officially listed supported version)        |
+| CUDA 11.4 – 11.8 | no matching GCC module in EESSI 2023.06                 |
+| CUDA < 11.4      | not supported                                           |
+
+To load correct ESSI module run, substituting x for the compatible GCC version:
+
+```[bash]
+module load GCC/12.x.0
+```
+
+Running `which gcc` will now give a path `.../2023.06/software...` which is the full compiler provided by EESSI. This is what we want for CUDA builds.
+
+Now you can run `cmake` and `make` to compile CUDA using EESSI's `gcc`.
+
+#### Test: Compile deviceQuery from CUDA-Samples
+
+To test that your EESSI setup can compile CUDA, try compiling `deviceQuery` from CUDA-Samples with the following steps:
+
+```[bash]
+git clone https://github.com/NVIDIA/cuda-samples.git
+cd cuda-samples/Samples/1_Utilities/deviceQuery
+mkdir -p build
+cd build
+cmake ..
+make
+./deviceQuery
+```