Skip to content

Commit 7b10ce8

Browse files
committed
Import jopa container
1 parent 47f7435 commit 7b10ce8

File tree

15 files changed

+710
-0
lines changed

15 files changed

+710
-0
lines changed

environments/jopa/README.md

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# JOPA Container
2+
3+
Contains the dependencies required to build mo-bundle and run JOPA
4+
5+
https://github.com/MetOffice/mo-bundle
6+
7+
# Building JOPA / mo-bundle
8+
9+
## NCI, with a container
10+
11+
1. Load the container
12+
13+
```
14+
module use /g/data/access/ngm
15+
module load envs/jopa
16+
```
17+
18+
2. Build Jopa inside the container
19+
20+
```
21+
cd mo-bundle
22+
23+
imagerun cmake --preset=bom-container --workflow
24+
```
25+
26+
## Met Office VDI, no container
27+
28+
1. Install Spack, and activate with `$SPACK/share/spack/setup-env.sh`
29+
30+
2. Copy `configs/meto_vdi/*` to the Spack configuration directory (e.g. `~/.spack` or `$SPACK/etc/spack`)
31+
32+
3. Install and setup a compiler, e.g. GCC 9
33+
34+
```
35+
spack install gcc@9
36+
spack compiler find $(spack find --format='{prefix}' gcc@9)
37+
```
38+
39+
4. Install the environment with GCC and MPICH
40+
41+
```
42+
export SPACK_COMPILER="gcc@9"
43+
export SPACK_MPI="mpich +slurm"
44+
45+
srun -q -n1 -c10 --mem=40G --time=120 env -u SLURM_NODELIST ./bin/install.sh jopa-v0
46+
```
47+
48+
5. Activate the environment
49+
50+
```
51+
spack load $SPACK_COMPILER
52+
spack env activate jopa-v0
53+
export SPACK_ENV_VIEW=$SPACK_ENV/.spack-env/view
54+
```
55+
56+
6. Build mo-bundle
57+
58+
```
59+
cd mo-bundle
60+
61+
cmake --preset=bom-container --workflow
62+
```

environments/jopa/post-install.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
# Copyright 2023 Bureau of Meteorology
3+
# Author Scott Wales
4+
5+
set -eu
6+
set -o pipefail
7+
8+
SCRIPT_DIR=$( cd -- "$( dirname -- "$(readlink -f ${BASH_SOURCE[0]})" )" &> /dev/null && pwd )
9+
10+
# Add FMS
11+
echo "CPATH=$(spack find --format '{prefix}' fms)/include_r4:\$CPATH" >> $SPACK_ROOT/bin/activate-full.sh

environments/jopa/spack.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
spack:
2+
specs:
3+
- eckit@1.24.4 compression=lz4
4+
- ecmwf-atlas@0.34.0
5+
- fckit@0.11.0
6+
- ectrans@1.2.0
7+
- fiat@1.1.2
8+
- odc
9+
10+
- openblas
11+
- boost
12+
- 'cmake@3.26:'
13+
- ecbuild@3.7
14+
- eigen
15+
- git-lfs
16+
- gsl-lite
17+
- hdf5-plugins
18+
- jedi-cmake
19+
- lapack
20+
- netcdf-c
21+
- netcdf-cxx4
22+
- netcdf-fortran
23+
- ninja
24+
- nlohmann-json-schema-validator
25+
- nlohmann-json
26+
- py-pybind11
27+
- shumlib
28+
- udunits
29+
- zstd
30+
- bom-ngm.hdf5@1.12.2 # Must match mamba
31+
- fms@release-jcsda
32+
33+
packages:
34+
git-lfs:
35+
require: '%gcc'
36+
go:
37+
require: '%gcc'
38+
openmpi:
39+
require: '~static +cxx'

spack/packages/ecbuild/package.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
2+
# Spack Project Developers. See the top-level COPYRIGHT file for details.
3+
#
4+
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
5+
6+
from spack.package import *
7+
8+
9+
class Ecbuild(CMakePackage):
10+
"""ecBuild is the ECMWF build system. It is built on top of CMake and
11+
consists of a set of macros as well as a wrapper around CMake"""
12+
13+
homepage = "https://github.com/ecmwf/ecbuild"
14+
url = "https://github.com/ecmwf/ecbuild/archive/refs/tags/3.6.1.tar.gz"
15+
16+
maintainers("skosukhin")
17+
18+
version(
19+
"3.7.2",
20+
sha256="7a2d192cef1e53dc5431a688b2e316251b017d25808190faed485903594a3fb9",
21+
)
22+
version(
23+
"3.6.5",
24+
sha256="98bff3d3c269f973f4bfbe29b4de834cd1d43f15b1c8d1941ee2bfe15e3d4f7f",
25+
)
26+
version(
27+
"3.6.1",
28+
sha256="796ccceeb7af01938c2f74eab0724b228e9bf1978e32484aa3e227510f69ac59",
29+
)
30+
31+
depends_on("cmake@3.11:", type=("build", "run"))
32+
33+
# See https://github.com/ecmwf/ecbuild/issues/35
34+
depends_on("cmake@:3.19", type=("build", "run"), when="@:3.6.1")
35+
36+
# Some of the installed scripts require running Perl:
37+
depends_on("perl", type=("build", "run"))

spack/packages/eckit/package.py

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
2+
# Spack Project Developers. See the top-level COPYRIGHT file for details.
3+
#
4+
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
5+
6+
from spack.package import *
7+
8+
9+
class Eckit(CMakePackage):
10+
"""ecKit is a cross-platform c++ toolkit that supports development of tools
11+
and applications at ECMWF."""
12+
13+
homepage = "https://github.com/ecmwf/eckit"
14+
url = "https://github.com/ecmwf/eckit/archive/refs/tags/1.16.0.tar.gz"
15+
16+
maintainers("skosukhin")
17+
18+
version(
19+
"1.24.4",
20+
sha256="b6129eb4f7b8532aa6905033e4cf7d09aadc8547c225780fea3db196e34e4671",
21+
)
22+
version(
23+
"1.24.0",
24+
sha256="a25440011c66decbfb4908e5f4b7d0025b2166acfb9db941abdeb38d477c45b3",
25+
)
26+
version(
27+
"1.20.2",
28+
sha256="9c11ddaaf346e40d11312b81ca7f1b510017f26618f4c0f5c5c59c37623fbac8",
29+
)
30+
version(
31+
"1.19.0",
32+
sha256="a5fef36b4058f2f0aac8daf5bcc9740565f68da7357ddd242de3a5eed4765cc7",
33+
)
34+
version(
35+
"1.16.3",
36+
sha256="d2aae7d8030e2ce39e5d04e36dd6aa739f3c8dfffe32c61c2a3127c36b573485",
37+
)
38+
version(
39+
"1.16.0",
40+
sha256="9e09161ea6955df693d3c9ac70131985eaf7cf24a9fa4d6263661c6814ebbaf1",
41+
)
42+
43+
variant("tools", default=True, description="Build the command line tools")
44+
variant("mpi", default=True, description="Enable MPI support")
45+
variant(
46+
"admin", default=True, description="Build utilities for administration tools"
47+
)
48+
variant("sql", default=True, description="Build SQL engine")
49+
variant(
50+
"linalg",
51+
values=any_combination_of("eigen", "armadillo", "mkl", "lapack"),
52+
description="List of supported linear algebra backends",
53+
)
54+
variant(
55+
"compression",
56+
values=any_combination_of("bzip2", "snappy", "lz4", "aec"),
57+
description="List of supported compression backends",
58+
)
59+
variant("xxhash", default=True, description="Enable xxHash support for hashing")
60+
variant(
61+
"ssl", default=False, description="Enable MD4 and SHA1 support with OpenSSL"
62+
)
63+
variant("curl", default=False, description="Enable URL data transferring with cURL")
64+
variant(
65+
"jemalloc", default=False, description="Link against jemalloc memory allocator"
66+
)
67+
variant(
68+
"unicode",
69+
default=True,
70+
description="Enable support for Unicode characters in Yaml/JSON" "parsers",
71+
)
72+
variant("aio", default=True, description="Enable asynchronous IO")
73+
74+
depends_on("cmake@3.12:", type="build")
75+
depends_on("ecbuild@3.5:", type="build")
76+
77+
depends_on("mpi", when="+mpi")
78+
79+
depends_on("yacc", type="build", when="+admin")
80+
depends_on("flex", type="build", when="+admin")
81+
depends_on("ncurses", when="+admin")
82+
83+
depends_on("yacc", type="build", when="+sql")
84+
depends_on("flex", type="build", when="+sql")
85+
depends_on("python@3", type="build")
86+
87+
depends_on("eigen", when="linalg=eigen")
88+
depends_on("armadillo", when="linalg=armadillo")
89+
depends_on("mkl", when="linalg=mkl")
90+
depends_on("lapack", when="linalg=lapack")
91+
92+
depends_on("bzip2", when="compression=bzip2")
93+
depends_on("snappy", when="compression=snappy")
94+
depends_on("lz4", when="compression=lz4")
95+
depends_on("libaec", when="compression=aec")
96+
97+
depends_on("openssl", when="+ssl")
98+
99+
depends_on("curl", when="+curl")
100+
101+
depends_on("jemalloc", when="+jemalloc")
102+
103+
# The package enables LAPACK backend (together with MKL backend)
104+
# when='linalg=mkl'. This leads to two identical installations when:
105+
# eckit linalg=mkl
106+
# eckit linalg=mkl,lapack
107+
# We prevent that by introducing the following conflict:
108+
conflicts(
109+
"linalg=lapack",
110+
when="linalg=mkl",
111+
msg='"linalg=lapack" is implied when "linalg=mkl" and '
112+
"must not be specified additionally",
113+
)
114+
115+
def cmake_args(self):
116+
args = [
117+
# Some features that we want to build are experimental:
118+
self.define("ENABLE_EXPERIMENTAL", self._enable_experimental),
119+
self.define_from_variant("ENABLE_BUILD_TOOLS", "tools"),
120+
# We let ecBuild find the MPI library. We could help it by setting
121+
# CMAKE_C_COMPILER to mpicc but that might give CMake a wrong
122+
# impression that no additional flags are needed to link to
123+
# libpthread, which will lead to problems with libraries that are
124+
# linked with the C++ compiler. We could additionally set
125+
# CMAKE_CXX_COMPILER to mpicxx. That would solve the problem with
126+
# libpthread but lead to overlinking to MPI libraries, which we
127+
# currently prefer to avoid since ecBuild does the job in all known
128+
# cases.
129+
self.define_from_variant("ENABLE_MPI", "mpi"),
130+
self.define_from_variant("ENABLE_ECKIT_CMD", "admin"),
131+
self.define_from_variant("ENABLE_ECKIT_SQL", "sql"),
132+
self.define("ENABLE_EIGEN", "linalg=eigen" in self.spec),
133+
self.define("ENABLE_ARMADILLO", "linalg=armadillo" in self.spec),
134+
self.define("ENABLE_MKL", "linalg=mkl" in self.spec),
135+
self.define("ENABLE_BZIP2", "compression=bzip2" in self.spec),
136+
self.define("ENABLE_SNAPPY", "compression=snappy" in self.spec),
137+
self.define("ENABLE_LZ4", "compression=lz4" in self.spec),
138+
self.define("ENABLE_AEC", "compression=aec" in self.spec),
139+
self.define_from_variant("ENABLE_XXHASH", "xxhash"),
140+
self.define_from_variant("ENABLE_SSL", "ssl"),
141+
self.define_from_variant("ENABLE_CURL", "curl"),
142+
self.define_from_variant("ENABLE_JEMALLOC", "jemalloc"),
143+
self.define_from_variant("ENABLE_UNICODE", "unicode"),
144+
self.define_from_variant("ENABLE_AIO", "aio"),
145+
self.define("ENABLE_TESTS", self.run_tests),
146+
# Unconditionally disable additional unit/performance tests, since
147+
# they download additional data (~1.6GB):
148+
self.define("ENABLE_EXTRA_TESTS", False),
149+
# No reason to check for doxygen and generate the documentation
150+
# since it is not installed:
151+
self.define("ENABLE_DOCS", False),
152+
# Disable features that are currently not needed:
153+
self.define("ENABLE_CUDA", False),
154+
self.define("ENABLE_VIENNACL", False),
155+
# Ceph/Rados storage support requires https://github.com/ceph/ceph
156+
# and will be added later:
157+
self.define("ENABLE_RADOS", False),
158+
# rsync support requires https://github.com/librsync/librsync and
159+
# will be added later:
160+
self.define("ENABLE_RSYNC", False),
161+
# Disable "prototyping code that may never see the light of day":
162+
self.define("ENABLE_SANDBOX", False),
163+
self.define(
164+
"PYTHON_EXECUTABLE", self.spec["python"].prefix + "/bin/python3"
165+
),
166+
]
167+
168+
if "linalg=mkl" not in self.spec:
169+
# ENABLE_LAPACK is ignored if MKL backend is enabled
170+
# (the LAPACK backend is still built though):
171+
args.append(self.define("ENABLE_LAPACK", "linalg=lapack" in self.spec))
172+
173+
if "+admin" in self.spec and "+termlib" in self.spec["ncurses"]:
174+
# Make sure that libeckit_cmd is linked to a library that resolves 'setupterm',
175+
# 'tputs', etc. That is either libncurses (when 'ncurses~termlib') or libtinfo (when
176+
# 'ncurses+termlib'). CMake considers the latter only if CURSES_NEED_NCURSES is set to
177+
# TRUE. Note that the installation of eckit does not fail without this but the building
178+
# of a dependent package (e.g. fdb) might fail due to the undefined references.
179+
args.append(self.define("CURSES_NEED_NCURSES", True))
180+
181+
return args
182+
183+
def check(self):
184+
ctest_args = ["-j", str(make_jobs)]
185+
186+
broken_tests = []
187+
if self._enable_experimental:
188+
# The following test quasi-randomly fails not because it reveals a bug in the library
189+
# but because its implementation has a bug (static initialization order fiasco):
190+
broken_tests.append("eckit_test_experimental_singleton_singleton")
191+
192+
if broken_tests:
193+
ctest_args.extend(["-E", "|".join(broken_tests)])
194+
195+
with working_dir(self.build_directory):
196+
ctest(*ctest_args)
197+
198+
@property
199+
def _enable_experimental(self):
200+
return "linalg=armadillo" in self.spec
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
From a27fc304525607c16cae7a44a2424c328df34d30 Mon Sep 17 00:00:00 2001
2+
From: Willem Deconinck <willem.deconinck@ecmwf.int>
3+
Date: Wed, 10 May 2023 13:44:58 +0200
4+
Subject: [PATCH] Fix compilation of atlas_io due to missing header <cstdint>
5+
6+
---
7+
atlas_io/src/atlas_io/Data.h | 2 ++
8+
1 file changed, 2 insertions(+)
9+
10+
diff --git a/atlas_io/src/atlas_io/Data.h b/atlas_io/src/atlas_io/Data.h
11+
index 7431fd7f4..69c51096f 100644
12+
--- a/atlas_io/src/atlas_io/Data.h
13+
+++ b/atlas_io/src/atlas_io/Data.h
14+
@@ -10,6 +10,8 @@
15+
16+
#pragma once
17+
18+
+#include <cstdint>
19+
+
20+
#include "eckit/io/Buffer.h"
21+
22+
namespace atlas {

0 commit comments

Comments
 (0)