Skip to content

Commit f510258

Browse files
assumptionsandgassumptionsandg
committed
OFED workflow
1 parent 2e40537 commit f510258

File tree

5 files changed

+336
-0
lines changed

5 files changed

+336
-0
lines changed

‎.github/workflows/package-build-ofed.yml

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
---
2+
name: Build OFED packages
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
rocky9:
7+
description: Build Rocky Linux 9
8+
type: boolean
9+
default: true
10+
secrets:
11+
KAYOBE_VAULT_PASSWORD:
12+
required: true
13+
CLOUDS_YAML:
14+
required: true
15+
OS_APPLICATION_CREDENTIAL_ID:
16+
required: true
17+
OS_APPLICATION_CREDENTIAL_SECRET:
18+
required: true
19+
20+
env:
21+
ANSIBLE_FORCE_COLOR: True
22+
KAYOBE_ENVIRONMENT: ci-builder
23+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
24+
jobs:
25+
overcloud-ofed-packages:
26+
name: Build OFED packages
27+
if: github.repository == 'stackhpc/stackhpc-kayobe-config'
28+
runs-on: arc-skc-host-image-builder-runner
29+
permissions: {}
30+
steps:
31+
- name: Install Package
32+
uses: ConorMacBride/install-package@main
33+
with:
34+
apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq
35+
36+
- name: Start the SSH service
37+
run: |
38+
sudo /etc/init.d/ssh start
39+
40+
- name: Checkout
41+
uses: actions/checkout@v4
42+
with:
43+
path: src/kayobe-config
44+
45+
- name: Determine OpenStack release
46+
id: openstack_release
47+
run: |
48+
BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
49+
echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
50+
51+
- name: Clone StackHPC Kayobe repository
52+
uses: actions/checkout@v4
53+
with:
54+
repository: stackhpc/kayobe
55+
ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
56+
path: src/kayobe
57+
58+
- name: Install Kayobe
59+
run: |
60+
mkdir -p venvs &&
61+
pushd venvs &&
62+
python3 -m venv kayobe &&
63+
source kayobe/bin/activate &&
64+
pip install -U pip &&
65+
pip install ../src/kayobe
66+
67+
- name: Install terraform
68+
uses: hashicorp/setup-terraform@v2
69+
70+
- name: Initialise terraform
71+
run: terraform init
72+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
73+
74+
- name: Generate SSH keypair
75+
run: ssh-keygen -f id_rsa -N ''
76+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
77+
78+
- name: Generate clouds.yaml
79+
run: |
80+
cat << EOF > clouds.yaml
81+
${{ secrets.CLOUDS_YAML }}
82+
EOF
83+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
84+
85+
- name: Generate terraform.tfvars
86+
run: |
87+
cat << EOF > terraform.tfvars
88+
ssh_public_key = "id_rsa.pub"
89+
ssh_username = "rocky"
90+
aio_vm_name = "skc-host-image-builder"
91+
# Must be a Rocky Linux 9 host to successfully build all images
92+
# This MUST NOT be an LVM image. It can cause confusing conficts with the built image.
93+
aio_vm_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2"
94+
aio_vm_flavor = "en1.medium"
95+
aio_vm_network = "stackhpc-ci"
96+
aio_vm_subnet = "stackhpc-ci"
97+
aio_vm_interface = "eth0"
98+
EOF
99+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
100+
101+
- name: Terraform Plan
102+
run: terraform plan
103+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
104+
env:
105+
OS_CLOUD: "openstack"
106+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
107+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
108+
109+
- name: Terraform Apply
110+
run: |
111+
for attempt in $(seq 5); do
112+
if terraform apply -auto-approve; then
113+
echo "Created infrastructure on attempt $attempt"
114+
exit 0
115+
fi
116+
echo "Failed to create infrastructure on attempt $attempt"
117+
sleep 10
118+
terraform destroy -auto-approve
119+
sleep 60
120+
done
121+
echo "Failed to create infrastructure after $attempt attempts"
122+
exit 1
123+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
124+
env:
125+
OS_CLOUD: "openstack"
126+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
127+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
128+
129+
- name: Get Terraform outputs
130+
id: tf_outputs
131+
run: |
132+
terraform output -json
133+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
134+
135+
- name: Write Terraform outputs
136+
run: |
137+
cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml
138+
${{ steps.tf_outputs.outputs.stdout }}
139+
EOF
140+
141+
- name: Write Terraform network config
142+
run: |
143+
cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml
144+
---
145+
aio_ips:
146+
builder: "{{ access_ip_v4.value }}"
147+
EOF
148+
149+
- name: Write Terraform network interface config
150+
run: |
151+
mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed
152+
rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
153+
cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
154+
admin_interface: "{{ access_interface.value }}"
155+
aio_interface: "{{ access_interface.value }}"
156+
EOF
157+
158+
- name: Manage SSH keys
159+
run: |
160+
mkdir -p ~/.ssh
161+
touch ~/.ssh/authorized_keys
162+
cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys
163+
cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/
164+
165+
- name: Bootstrap the control host
166+
run: |
167+
source venvs/kayobe/bin/activate &&
168+
source src/kayobe-config/kayobe-env --environment ci-builder &&
169+
kayobe control host bootstrap
170+
171+
- name: Configure the seed host (Builder VM)
172+
run: |
173+
source venvs/kayobe/bin/activate &&
174+
source src/kayobe-config/kayobe-env --environment ci-builder &&
175+
kayobe seed host configure -e seed_bootstrap_user=rocky --skip-tags network
176+
177+
- name: Run OFED builder playbook
178+
run: |
179+
source venvs/kayobe/bin/activate &&
180+
source src/kayobe-config/kayobe-env --environment ci-builder &&
181+
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed.yml
182+
env:
183+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }
184+
185+
- name: Run OFED publish playbook
186+
run: |
187+
source venvs/kayobe/bin/activate &&
188+
source src/kayobe-config/kayobe-env --environment ci-builder &&
189+
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
190+
env:
191+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }
192+
193+
- name: Destroy
194+
run: terraform destroy -auto-approve
195+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
196+
env:
197+
OS_CLOUD: openstack
198+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
199+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
200+
if: always()

‎etc/kayobe/ansible/build-ofed.yml

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
---
2+
- name: Build OFED packages
3+
hosts: ofed-builder
4+
gather_facts: false
5+
vars:
6+
stackhpc_mlnx_ofed_file_string: MLNX_OFED_LINUX-{{ stackhpc_pulp_mlnx_ofed_version }}-rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}-x86_64
7+
tasks:
8+
- name: Install package dependencies
9+
ansible.builtin.dnf:
10+
name:
11+
- kpartx
12+
- cloud-init
13+
- perl
14+
- rpm-build
15+
- automake
16+
- patch
17+
- kernel
18+
- kernel-devel
19+
- autoconf
20+
- pciutils
21+
- kernel-rpm-macros
22+
- lsof
23+
- libtool
24+
- tk
25+
- gcc-gfortran
26+
- tcl
27+
state: latest
28+
update_cache: true
29+
become: true
30+
31+
- name: Reboot builder to apply kernel update
32+
ansible.builtin.reboot:
33+
reboot_timeout: 600
34+
35+
- name: Install python dependencies
36+
ansible.builtin.pip:
37+
name: pulp-cli
38+
39+
- name: Create build directory
40+
ansible.builtin.file:
41+
path: /opt/ofed
42+
state: directory
43+
mode: 0777
44+
become: true
45+
46+
- name: Download MellanoxOFED archive
47+
ansible.builtin.get_url:
48+
url: https://content.mellanox.com/ofed/MLNX_OFED-{{ stackhpc_pulp_mlnx_ofed_version }}/{{ stackhpc_mlnx_ofed_file_string }}.tgz
49+
dest: /opt/ofed/ofed-archive
50+
51+
- name: Extract MellanoxOFED archive
52+
ansible.builtin.unarchive:
53+
src: /opt/ofed/ofed-archive
54+
dest: /opt/ofed
55+
56+
- name: Ensure the current kernel is supported
57+
ansible.builtin.shell:
58+
cmd: |
59+
/opt/ofed/{{ stackhpc_mlnx_ofed_file_string }}/mlnx_add_kernel_support.sh \
60+
--mlnx_ofed /opt/ofed/{{ stackhpc_mlnx_ofed_file_string }} \
61+
--make-tgz -y \
62+
become: true
63+
64+
- name: Extract the new archive
65+
ansible.builtin.unarchive:
66+
src: /tmp/{{ stackhpc_mlnx_ofed_file_string }}-ext.tgz
67+
dest: /opt/ofed
68+
become: true

‎etc/kayobe/ansible/push-ofed.yml

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
---
2+
- name: Push OFED packages
3+
hosts: ofed-builder
4+
gather_facts: false
5+
vars:
6+
stackhpc_mlnx_ofed_file_string: MLNX_OFED_LINUX-{{ stackhpc_pulp_mlnx_ofed_version }}-rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}-x86_64
7+
tasks:
8+
- name: Assign timestamp variable
9+
ansible.builtin.set_fact:
10+
stackhpc_pulp_repo_mlnx_ofed_version: "{{ ansible_date_time.iso8601_basic_short }}"
11+
12+
- name: Append timestamp to pulp-repo-versions
13+
ansible.builtin.lineinfile:
14+
path: "{{ lookup('env', KAYOBE_CONFIG_PATH) }}/pulp-repo-versions.yml"
15+
line: "stackhpc_pulp_repo_mlnx_ofed_version: {{ stackhpc_pulp_repo_mlnx_ofed_version }}"
16+
17+
- name: Create Pulp repository for OFED
18+
pulp.squeezer.rpm_repository:
19+
pulp_url: "{{ stackhpc_release_pulp_url }}"
20+
username: "{{ stackhpc_release_pulp_username }}"
21+
password: "{{ stackhpc_release_pulp_password }}"
22+
name: "{{ stackhpc_pulp_repo_mlnx_ofed.name }}"
23+
state: present
24+
retries: "{{ pulp_repository_rpm_repositories_retries }}"
25+
26+
- name: Upload OFED RPMs to Pulp
27+
ansible.builtin.shell:
28+
cmd: |
29+
pulp \
30+
--base-url '{{ stackhpc_release_pulp_url }}' \
31+
--username '{{ stackhpc_release_pulp_username }}' \
32+
--password '{{ stackhpc_release_pulp_password }}' \
33+
rpm content \
34+
--type package upload \
35+
--repository '{{ stackhpc_pulp_repo_mlnx_ofed.name }}' \
36+
--file {{ item }} \
37+
with_fileglob: "/opt/ofed/{{ stackhpc_mlnx_ofed_file_string }}-ext/RPMS/*.rpm"
38+
no_log: true
39+
40+
- name: Create Pulp publication for OFED
41+
pulp.squeezer.rpm_publication:
42+
pulp_url: "{{ stackhpc_release_pulp_url }}"
43+
username: "{{ stackhpc_release_pulp_username }}"
44+
password: "{{ stackhpc_release_pulp_password }}"
45+
repository: "{{ stackhpc_pulp_repo_mlnx_ofed.name }}"
46+
state: present
47+
48+
- name: Create Pulp distribution for OFED
49+
pulp.squeezer.rpm_distribution:
50+
pulp_url: "{{ stackhpc_release_pulp_url }}"
51+
username: "{{ stackhpc_release_pulp_username }}"
52+
password: "{{ stackhpc_release_pulp_password }}"
53+
name: "{{ stackhpc_pulp_repo_mlnx_ofed.distribution_name }}"
54+
base_path: "{{ stackhpc_pulp_repo_mlnx_ofed.base_path }}"

‎etc/kayobe/environments/ci-builder/inventory/hosts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
11
# A 'seed' host used for building images.
2+
3+
[ofed-builder:children]
4+
seed
5+
26
[seed]
37
builder

‎etc/kayobe/pulp.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,16 @@ stackhpc_pulp_repository_rpm_repos: >-
369369
{%- endfor -%}
370370
{{ rpm_repos }}
371371
372+
# Mellanox OFED version
373+
stackhpc_pulp_mlnx_ofed_version: 24.04-0.7.0.0
374+
375+
# Mellanox OFED repositories
376+
stackhpc_pulp_repo_mlnx_ofed:
377+
name: Mellanox Technologies mlnx_ofed {{ stackhpc_pulp_mlnx_ofed_version }}
378+
url: "{{ stackhpc_release_pulp_content_url }}/mlnx_ofed/{{ stackhpc_pulp_mlnx_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_pulp_repo_mlnx_ofed_version }}"
379+
distribution_name: "mlnx_ofed_{{ stackhpc_pulp_mlnx_ofed_version }}-"
380+
base_path: "mlnx_ofed/{{ stackhpc_pulp_mlnx_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/"
381+
372382
# Publication format is a subset of distribution.
373383
stackhpc_pulp_publication_rpm_development: "{{ stackhpc_pulp_distribution_rpm_development }}"
374384

0 commit comments

Comments
 (0)