Skip to content

Commit b314737

Browse files
rickgrubin-tomorrowrickgrubin-tomorrow
authored
Merge branch 'JCSDA:develop' into develop
2 parents 67955c5 + 99271d9 commit b314737

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+790
-683
lines changed

‎.github/workflows/macos-ci-aarch64.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ jobs:
103103
104104
# Concretize and check for duplicates
105105
spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3-buildcache
106-
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3-buildcache -i fms -i crtm -i crtm-fix -i esmf -i mapl
106+
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl
107107
108108
# Add and update source cache
109109
spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/
@@ -172,7 +172,7 @@ jobs:
172172
173173
# Concretize and check for duplicates
174174
spack concretize --force 2>&1 | tee log.concretize.apple-clang-14.0.3
175-
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i crtm-fix -i esmf -i mapl
175+
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl
176176
177177
# Add binary cache back in
178178
spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/

‎.github/workflows/ubuntu-ci-containers-x86_64.yaml‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
name: ubuntu-ci-container-x86_64-build
22
on:
33
# Uncomment this to test for PRs (but do not submit)
4-
#pull_request:
4+
# pull_request:
55
# paths-ignore:
66
# - 'configs/sites/**'
77
# - 'doc/**'
88
# - '**.md'
99
# - '.github/ISSUE_TEMPLATE/*'
1010
# - '.gitignore'
11-
#
11+
1212
schedule:
1313
- cron: '0 8 * * *'
1414
workflow_dispatch:
@@ -57,11 +57,14 @@ jobs:
5757
if [[ $DOW == 1 || $DOW == 4 ]]; then
5858
export CONTAINER=${{ inputs.container || 'docker-ubuntu-clang-mpich' }}
5959
export SPECS=${{ inputs.specs || 'jedi-ci' }}
60-
elif [[ $DOW == 2 || $DOW == 5 ]]; then
60+
elif [[ $DOW == 2 ]]; then
6161
export CONTAINER=${{ inputs.container || 'docker-ubuntu-gcc-openmpi' }}
6262
export SPECS=${{ inputs.specs || 'jedi-ci' }}
63+
elif [[ $DOW == 5 ]]; then
64+
export CONTAINER=${{ inputs.container || 'docker-ubuntu-gcc11-openmpi' }}
65+
export SPECS=${{ inputs.specs || 'jedi-ci' }}
6366
elif [[ $DOW == 3 || $DOW == 6 ]]; then
64-
export CONTAINER=${{ inputs.container || 'docker-ubuntu-intel-impi' }}
67+
export CONTAINER=${{ inputs.container || 'docker-ubuntu-oneapi-impi' }}
6568
export SPECS=${{ inputs.specs || 'jedi-ci' }}
6669
else
6770
# Day 7: The Sabbath of rest (but do some house keeping later on)

‎.github/workflows/ubuntu-ci-x86_64-gnu.yaml‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,14 +51,16 @@ jobs:
5151
# Set up spack-stack
5252
source ./setup.sh
5353
54-
declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev")
54+
declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev" "neptune-dev")
5555
for TEMPLATE in "${TEMPLATES[@]}"; do
5656
if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then
5757
export ENVNAME=ue-gcc-11.4.0-buildcache
5858
elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then
5959
export ENVNAME=se-gcc-11.4.0-buildcache
6060
elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then
6161
export ENVNAME=ce-gcc-11.4.0-buildcache
62+
elif [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then
63+
export ENVNAME=ne-gcc-11.4.0-buildcache
6264
fi
6365
echo "Creating environment ${ENVNAME} from template ${TEMPLATE}"
6466
@@ -97,9 +99,14 @@ jobs:
9799
# Add additional variants for MET packages, different from config/common/packages.yaml
98100
spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
99101
102+
# Turn off variant xnrl for neptune-python-env because xnrl is not publicly available
103+
if [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then
104+
sed -i 's/+xnrl/~xnrl/g' ${ENVDIR}/spack.yaml
105+
fi
106+
100107
# Concretize and check for duplicates
101108
spack concretize 2>&1 | tee log.concretize.gnu-11.4.0-buildcache
102-
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0-buildcache -i fms -i crtm -i crtm-fix -i esmf -i mapl
109+
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl
103110
104111
# Add and update source cache
105112
spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
@@ -170,7 +177,7 @@ jobs:
170177
171178
# Concretize and check for duplicates
172179
spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0
173-
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i crtm-fix -i esmf -i mapl
180+
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl
174181
175182
# Add binary cache back in
176183
spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/

‎.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ jobs:
142142
143143
# Concretize and check for duplicates
144144
spack concretize 2>&1 | tee log.concretize.oneifx-2024.2.0-buildcache
145-
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneifx-2024.2.0-buildcache -i fms -i crtm -i crtm-fix -i esmf -i mapl
145+
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl
146146
147147
# Add and update source cache
148148
spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
@@ -182,6 +182,9 @@ jobs:
182182
spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneifx-2024.2.0-buildcache.unified-env
183183
spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
184184
185+
# Check for libirc.so in shared libraries (returns 1=error if found)
186+
${SPACK_STACK_DIR}/util/check_libirc.sh
187+
185188
# Remove binary cache for next round of concretization
186189
if [ "$USE_BINARY_CACHE" = true ] ; then
187190
spack mirror rm local-binary
@@ -209,7 +212,7 @@ jobs:
209212
210213
# Concretize and check for duplicates
211214
spack concretize --force 2>&1 | tee log.concretize.oneifx-2024.2.0
212-
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneifx-2024.2.0 -i fms -i crtm -i crtm-fix -i esmf -i mapl
215+
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl
213216
214217
# Add binary cache back in
215218
spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
@@ -222,6 +225,9 @@ jobs:
222225
# Check shared libraries
223226
${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
224227
228+
# Check for libirc.so in shared libraries (returns 1=error if found)
229+
${SPACK_STACK_DIR}/util/check_libirc.sh
230+
225231
# Create modules
226232
spack clean -a
227233
spack module tcl refresh -y

‎.github/workflows/ubuntu-ci-x86_64-oneapi.yaml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ jobs:
142142
143143
# Concretize and check for duplicates
144144
spack concretize 2>&1 | tee log.concretize.oneapi-2024.2.0-buildcache
145-
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0-buildcache -i fms -i crtm -i crtm-fix -i esmf -i mapl
145+
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl
146146
147147
# Add and update source cache
148148
spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
@@ -182,6 +182,9 @@ jobs:
182182
spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env
183183
spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
184184
185+
# Check for libirc.so in shared libraries (returns 1=error if found)
186+
${SPACK_STACK_DIR}/util/check_libirc.sh
187+
185188
# Remove binary cache for next round of concretization
186189
if [ "$USE_BINARY_CACHE" = true ] ; then
187190
spack mirror rm local-binary
@@ -209,7 +212,7 @@ jobs:
209212
210213
# Concretize and check for duplicates
211214
spack concretize --force 2>&1 | tee log.concretize.oneapi-2024.2.0
212-
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0 -i fms -i crtm -i crtm-fix -i esmf -i mapl
215+
${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl
213216
214217
# Add binary cache back in
215218
spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
@@ -222,6 +225,9 @@ jobs:
222225
# Check shared libraries
223226
${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
224227
228+
# Check for libirc.so in shared libraries (returns 1=error if found)
229+
${SPACK_STACK_DIR}/util/check_libirc.sh
230+
225231
# Create modules
226232
spack clean -a
227233
spack module tcl refresh -y

‎configs/common/modules_lmod.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,10 @@ modules:
134134
^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug'
135135
^esmf@8.8.0~debug snapshot=none: 'esmf-8.8.0'
136136
^esmf@8.8.0+debug snapshot=none: 'esmf-8.8.0-debug'
137-
^esmf@8.9.0b05~debug snapshot=b05: 'esmf-8.9.0b05'
138-
^esmf@8.9.0b05+debug snapshot=b05: 'esmf-8.9.0b05-debug'
137+
^esmf@8.9.0b08~debug snapshot=b08: 'esmf-8.9.0b08'
138+
^esmf@8.9.0b08+debug snapshot=b08: 'esmf-8.9.0b08-debug'
139+
^esmf@8.9.0b09~debug snapshot=b09: 'esmf-8.9.0b09'
140+
^esmf@8.9.0b09+debug snapshot=b09: 'esmf-8.9.0b09-debug'
139141
openmpi:
140142
environment:
141143
set:

‎configs/common/modules_tcl.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,10 @@ modules:
132132
^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug'
133133
^esmf@8.8.0~debug snapshot=none: 'esmf-8.8.0'
134134
^esmf@8.8.0+debug snapshot=none: 'esmf-8.8.0-debug'
135-
^esmf@8.9.0b05~debug snapshot=b05: 'esmf-8.9.0b05'
136-
^esmf@8.9.0b05+debug snapshot=b05: 'esmf-8.9.0b05-debug'
135+
^esmf@8.9.0b08~debug snapshot=b08: 'esmf-8.9.0b08'
136+
^esmf@8.9.0b08+debug snapshot=b08: 'esmf-8.9.0b08-debug'
137+
^esmf@8.9.0b09~debug snapshot=b09: 'esmf-8.9.0b09'
138+
^esmf@8.9.0b09+debug snapshot=b09: 'esmf-8.9.0b09-debug'
137139
openmpi:
138140
environment:
139141
set:

‎configs/common/packages.yaml‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -70,17 +70,19 @@ packages:
7070
esmf:
7171
require:
7272
- '~xerces ~pnetcdf +shared +external-parallelio'
73-
- any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0b05 snapshot=b05']
73+
- any_of: ['@=8.6.1 snapshot=none', '@=8.8.0 snapshot=none', '@=8.9.0b09 snapshot=b09']
7474
- any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
7575
when: "%intel"
7676
message: "Extra ESMF compile options for Intel"
77-
- '+python'
78-
#- any_of: ['']
79-
# when: "%gcc"
80-
# message: "Extra ESMF compile options for GCC"
81-
#- any_of: ['']
82-
# when: "%apple-clang"
83-
# message: "Extra ESMF compile options for GCC"
77+
- any_of: ['+python']
78+
when: "%gcc"
79+
- any_of: ['+python']
80+
when: "%intel"
81+
- any_of: ['+python']
82+
when: "%oneapi"
83+
- any_of: ['~python']
84+
when: "%clang"
85+
message: "Disable python variant for ESMF because mpi4py doesn't build with clang"
8486
# To avoid duplicate packages (concretizer bug?)
8587
expat:
8688
require:
@@ -141,12 +143,12 @@ packages:
141143
hdf:
142144
require: '@4.2.15 +external-xdr ~fortran ~netcdf'
143145
hdf5:
144-
require: '@1.14.3 +hl +fortran +mpi +threadsafe ~szip'
146+
require: '@1.14.5 +hl +fortran +mpi +threadsafe ~szip'
145147
# Newer versions of hdf-eos2 require manual downloading, avoid
146148
hdf-eos2:
147149
require: '@2.20v1.00'
148150
ip:
149-
require: '@5.3.0 precision=4,d,8'
151+
require: '@5.4.0 precision=4,d,8'
150152
ip2:
151153
require: '@1.1.2'
152154
jasper:
@@ -170,7 +172,10 @@ packages:
170172
require: '@2.53.0 ~shared ~f2py'
171173
variants: '+pflogger'
172174
met:
173-
require: '@12.0.1 +python +grib2'
175+
require:
176+
- '@12.0.1'
177+
- '+python'
178+
- '+grib2'
174179
metplus:
175180
require: '@6.0.0'
176181
metis:
@@ -198,22 +203,19 @@ packages:
198203
require: '@4.6.1 +parallel_tests'
199204
# ninja - when adding information here, also check Discover site config
200205
odc:
201-
require: '@1.5.2 ~fortran'
202-
# Todo: update to 0.3.28 so that it builds with 'clang'. Check
203-
# packages_clang.yaml where we are overwriting openblas for now.
204-
openblas:
205-
require:
206-
- '@0.3.24'
206+
require: '@1.6.1 ~fortran'
207207
openmpi:
208208
require: '~internal-hwloc +two_level_namespace'
209209
openssl:
210210
require: '+shared'
211211
p4est:
212212
require: '@2.8.7'
213213
parallelio:
214-
require: '@2.6.2 +pnetcdf'
214+
require:
215+
- '@2.6.2'
215216
parallel-netcdf:
216-
require: '@1.12.3'
217+
require:
218+
- '@1.12.3'
217219
pflogger:
218220
require: '+mpi'
219221
pixman:
@@ -272,8 +274,10 @@ packages:
272274
require: '@69'
273275
py-setuptools-rust:
274276
require: '+rust_bootstrap'
277+
# When making changes here, also check the tier2/blackpearl/packages.yaml
275278
py-torch:
276-
require: '+custom-protobuf ~mkldnn'
279+
require:
280+
- '+custom-protobuf ~mkldnn'
277281
# To avoid duplicate packages
278282
py-urllib3:
279283
require: '@1.26.12'

‎configs/common/packages_clang.yaml‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,3 @@ packages:
1313
py-numpy:
1414
require:
1515
- '^openblas'
16-
# clang@19 requires openblas@0.3.28 - with this, we can'this
17-
# build all the Python dependencies as of 2024/12/11 due to
18-
# version conflicts.
19-
openblas:
20-
require::
21-
- '@0.3.28'
22-
# ip@5 doesn't build with clang@19 compilers (flang-new)
23-
# https://github.com/NOAA-EMC/NCEPLIBS-ip/issues/263
24-
# but requiring %gcc doesn't work when using openblas
25-
#ip:
26-
# require:
27-
# #- '~openmp'
28-
# - '%gcc'

‎configs/common/packages_oneapi.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,12 @@ packages:
2424
require: '%gcc'
2525
libbsd:
2626
require: '%gcc'
27+
met:
28+
require:
29+
- '+shared-intel'
30+
parallel-netcdf:
31+
require:
32+
- '+shared-intel'
2733
qt:
2834
require:
2935
- '%gcc'

0 commit comments

Comments
 (0)