From 427fb58711c7c28dfec4ba2be398d36a903dafe4 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 5 Jun 2024 14:58:00 +0200 Subject: [PATCH] Add custom playbooks to manage Dell firmware --- .../operations/dell-firmware-update.rst | 161 ++++++++++++++++++ doc/source/operations/index.rst | 1 + .../ansible/drac-firmware-inventory.yml | 34 ++++ etc/kayobe/ansible/drac-firmware-update.yml | 64 +++++++ etc/kayobe/ansible/requirements.yml | 2 + 5 files changed, 262 insertions(+) create mode 100644 doc/source/operations/dell-firmware-update.rst create mode 100644 etc/kayobe/ansible/drac-firmware-inventory.yml create mode 100644 etc/kayobe/ansible/drac-firmware-update.yml diff --git a/doc/source/operations/dell-firmware-update.rst b/doc/source/operations/dell-firmware-update.rst new file mode 100644 index 000000000..5d1d0fdd7 --- /dev/null +++ b/doc/source/operations/dell-firmware-update.rst @@ -0,0 +1,161 @@ +=============================== +Dell Firmware Update Automation +=============================== + +Overview +======== + +Custom playbooks are available to automate firmware updates on Dell hardware. + +We make use of `Dell Repository Manager (DRM) +`__. + +Prerequisites +============= + +DRM needs to listen on port 443 and needs access to the out-of-band management +network. Choose a host where it won't conflict with another service. + +To run DRM in a container, first start a container, that has a Docker volume to +host all the firmware files: + +.. code-block:: bash + + docker volume create dell_firmware + docker run --detach -v dell_firmware:/dell_firmware --name dell-drm --network host --restart always rockylinux:9.3 sleep infinity + +Copy in, and then run the installer: + +.. code-block:: bash + + curl -O https://dl.dell.com/FOLDER11468378M/1/DRMInstaller_3.4.5.938.bin -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/110.0' + docker cp DRMInstaller_3.4.5.938.bin dell-drm:/root + docker exec -it dell-drm bash + cd /root + chmod +x DRMInstaller_3.4.5.938.bin + ./DRMInstaller_3.4.5.938.bin + +Now you can run DRM, and download a new repo (customise the argument to +``--inputplatformlist`` depending on the targeted hardware): + +.. code-block:: bash + + /opt/dell/dellrepositorymanager/DRM_Service.sh & + drm --create -r=idrac_repo --inputplatformlist=R640,R6525 + drm --deployment-type=share --location=/dell_firmware -r=idrac_repo + +Note: sometimes the create call had to be run multiple times before it worked, +with errors relating to ``Unknown platform: R6525``. Restarting the service +might be required. + +Now we have the all the files in the Docker volume, we can start Apache to +expose the repo. Use this Dockerfile to support TLS: + +.. code-block:: dockerfile + + FROM httpd:2.4 + + RUN sed -i \ + -e 's/^#\(Include .*httpd-ssl.conf\)/\1/' \ + -e 's/^#\(LoadModule .*mod_ssl.so\)/\1/' \ + -e 's/^#\(LoadModule .*mod_socache_shmcb.so\)/\1/' \ + -e 's/Listen 80/#Listen 80/' \ + conf/httpd.conf + +Build a Docker image: + +.. code-block:: bash + + docker build --network host -t httpd:local . + +Generate a self-signed cert: + +.. code-block:: bash + + openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout apache.key -out apache.crt + +Run the container: + +.. code-block:: bash + + docker run -d --name dell-drm-web --network host -v dell_firmware:/usr/local/apache2/htdocs/ -v $PWD/apache.crt:/usr/local/apache2/conf/server.crt -v $PWD/apache.key:/usr/local/apache2/conf/server.key docker.io/library/httpd:local + +.. note:: + + At this point the repository may contain only old version of the firmwares. + Run an update once to make sure the latest files are available (see next + section). + +Updating the Repo +================= + +At a later date we will want to re-baseline to a new version. The repo +can be updated: + +.. code-block:: bash + + docker exec -it dell-drm bash + [root@seed /]# drm --update -r=idrac_repo + # check that it has iterated to a new version + [root@seed /]# drm -li=rep + + Listing Repositories... + + + Name Latest version Size Last modified date + ---- -------------- ---- ------------- + idrac_repo 1.01 4.82 GB 1/9/24 2:22 P.M + + # share the new version + [root@seed /]# drm --deployment-type=share --location=/dell_firmware -r=idrac_repo:1.01 + [root@seed /]# ls -ltra /dell_firmware | tail -1 + -rw-r--r-- 1 root root 7103842 Jan 9 14:24 idrac_repo_Catalog.xml + +Then update the ``dell_drm_repo`` variable in ``drac-firmware-update.yml`` if +required. + +Manually adding and update file +=============================== + +Clone an update package the windows format (the iDRAC knows how to process these): + +.. code-block:: bash + + curl 'https://dl.dell.com/FOLDER09614074M/2/Network_Firmware_77R8T_WN64_22.36.10.10.EXE?uid=39eab3c7-5ad6-4bfc-be6e-b9d09374accd&fn=Network_Firmware_77R8T_WN64_22.36.10.10.EXE' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/110.0' -O Network_Firmware_77R8T_WN64_22.36.10.10.EXE + +Import it into your repo: + +.. code-block:: bash + + drm --import --repository=idrac_repo:1.00 --source=/root --update-package="*.EXE" + +Export the repository: + +.. code-block:: bash + + drm --deployment-type=share --location=/dell_firmware -r=idrac_repo:1.02 + +Updating firmware versions on a Dell node +========================================= + +The updated firmware versions can be applied to a Dell node using the +``drac-firmware-update.yml`` playbook. + +The following command will show the list of firmware updates to be applied: + +.. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/drac-firmware-update.yml --limit + +The following command will apply firmware updates: + +.. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/drac-firmware-update.yml --limit -e dell_drm_apply_update=true + +.. note:: + + The playbook will likely fail with an error if the iDRAC firmware is being + updated, since this involves rebooting the iDRAC. Wait for the iDRAC to be + up and run the playbook again to ensure all firmwares have been updated + correctly. diff --git a/doc/source/operations/index.rst b/doc/source/operations/index.rst index 39f1bb847..9f40450cb 100644 --- a/doc/source/operations/index.rst +++ b/doc/source/operations/index.rst @@ -7,6 +7,7 @@ This guide is for operators of the StackHPC Kayobe configuration project. .. toctree:: :maxdepth: 1 + dell-firmware-update hotfix-playbook nova-compute-ironic octavia diff --git a/etc/kayobe/ansible/drac-firmware-inventory.yml b/etc/kayobe/ansible/drac-firmware-inventory.yml new file mode 100644 index 000000000..32b87e476 --- /dev/null +++ b/etc/kayobe/ansible/drac-firmware-inventory.yml @@ -0,0 +1,34 @@ +--- +- hosts: overcloud + gather_facts: false + connection: local + + collections: + - dellemc.openmanage + + vars: + secrets_ipmi_username: "{{ ipmi_admin_user }}" + secrets_ipmi_password: "{{ ipmi_admin_password }}" + drac_export_path: "{{ playbook_dir }}/idrac_firmware_inventory/" + ansible_python_interpreter: "{{ ansible_playbook_python }}" + tasks: + - name: Get Firmware inventory + idrac_firmware_info: + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + register: idrac_firmware_output + delegate_to: localhost + when: ipmi_address is defined + + - name: Ensure output dir exists + file: + state: directory + path: "{{ drac_export_path }}" + delegate_to: localhost + run_once: True + + - name: Write inventory to file + copy: + content: "{{ idrac_firmware_output.firmware_info }}" + dest: "{{ drac_export_path }}/{{ inventory_hostname }}" diff --git a/etc/kayobe/ansible/drac-firmware-update.yml b/etc/kayobe/ansible/drac-firmware-update.yml new file mode 100644 index 000000000..3ba348e6d --- /dev/null +++ b/etc/kayobe/ansible/drac-firmware-update.yml @@ -0,0 +1,64 @@ +--- +- hosts: overcloud + gather_facts: false + connection: local + serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}" + + collections: + - dellemc.openmanage + + vars: + # Run with ``-e dell_drm_apply_update=true`` to apply the firmware updates. + dell_drm_apply_update: False + dell_drm_address: "https://{{ oob_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}" + secrets_ipmi_username: "{{ ipmi_admin_user }}" + secrets_ipmi_password: "{{ ipmi_admin_password }}" + # Look at the README for more details, but you need a repo + # created in drm, and exported via a webserver: + # drm --create -r=idrac_repo --inputplatformlist=R640,R6525 + dell_drm_repo: "idrac_repo_Catalog.xml" + # Run with -e bifrost_maintenance=false if nodes have not been enroled yet + set_bifrost_maintenance: True + + tasks: + - name: Set maintenance mode + command: |- + docker exec bifrost_deploy bash -c ' + OS_CLOUD=bifrost openstack baremetal node maintenance set \ + --reason "Running drac-firmware-update.yml" \ + {{ inventory_hostname }}' + become: true + delegate_to: "{{ groups.seed.0 }}" + vars: + ansible_connection: ssh + ansible_host: "{{ hostvars[groups.seed.0].ansible_host }}" + when: set_bifrost_maintenance | bool + + - name: Update firmware from repository on a HTTP + idrac_firmware: + idrac_ip: "{{ ipmi_address }}" + idrac_user: "{{ secrets_ipmi_username }}" + idrac_password: "{{ secrets_ipmi_password }}" + reboot: True + job_wait: True + apply_update: "{{ dell_drm_apply_update | bool }}" + share_name: "{{ dell_drm_address }}" + catalog_file_name: "{{ dell_drm_repo }}" + ignore_cert_warning: True + register: idrac_firmware_output + delegate_to: localhost + when: ipmi_address is defined + - debug: + msg: "{{ idrac_firmware_output }}" + + - name: Unset maintenance mode + command: |- + docker exec bifrost_deploy bash -c ' + OS_CLOUD=bifrost openstack baremetal node maintenance unset \ + {{ inventory_hostname }}' + become: true + delegate_to: "{{ groups.seed.0 }}" + vars: + ansible_connection: ssh + ansible_host: "{{ hostvars[groups.seed.0].ansible_host }}" + when: set_bifrost_maintenance | bool diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index f1ae3bb63..ceb321efe 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,5 +1,7 @@ --- collections: + - name: dellemc.openmanage + version: 4.2.0 - name: stackhpc.cephadm version: 1.15.1 # NOTE: Pinning pulp.squeezer to 0.0.13 because 0.0.14+ depends on the