Includes CCPP scheme dependencies in the build (#282)

Uses updated CCPP-generated data to include scheme dependencies in the
CAM-SIMA build.

In draft until ESCOMP/atmospheric_physics#102 is
merged.

---------

Co-authored-by: Jesse Nusbaumer <nusbaume@ucar.edu>

Author: mattldawson

.dockerignore

@@ -9,3 +9,6 @@
 !test/
 !.config_files.xml
 !docker
+!bin/
+!.lib/
+!.gitmodules

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "ccpp-framework"]
         path          = ccpp_framework
         url           = https://github.com/NCAR/ccpp-framework
-        fxtag         = 2024-07-11-dev
+        fxtag         = 2024-07-19-dev
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework
 [submodule "mpas"]
@@ -14,7 +14,7 @@
 [submodule "ncar-physics"]
         path          = src/physics/ncar_ccpp
         url           = https://github.com/ESCOMP/atmospheric_physics
-        fxtag         = atmos_phys0_03_000
+        fxtag         = 098585940ad763be58ebab849bb8eaf325fda42a
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 [submodule "ccs_config"]

ccpp_framework

@@ -615,6 +615,11 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
         ufiles_str = datatable_report(cap_output_file, request, ";")
         utility_files = ufiles_str.split(';')
         _update_genccpp_dir(utility_files, genccpp_dir)
+        request = DatatableReport("dependencies")
+        dep_str = datatable_report(cap_output_file, request, ";")
+        if len(dep_str) > 0:
+            dependency_files = dep_str.split(';')
+            _update_genccpp_dir(dependency_files, genccpp_dir)
         ##XXgoldyXX: ^ Temporary fix:
     # End if
 

cime_config/cam_autogen.py

@@ -1,6 +1,6 @@
 # parts of CAM require x86 architecture (gptl, which relies on the rdtsc x86 assembly instruction)
 # esmf is am image you are expected to have built. Read the README file for instructions
-FROM esmf:latest
+FROM --platform=linux/amd64 esmf:latest
 
 ###################################################
 ## Install necessary packages
@@ -11,13 +11,18 @@ RUN dnf -y update \
       git \
       hostname \
       m4 \
-      python \
+      python39 \
+      pip \
       sudo \
       svn \
       tree \
       vim \
     && dnf clean all
 
+RUN ln -s $(which python3) /usr/bin/python && \
+    pip install --upgrade pip && \
+    pip install --upgrade setuptools
+
 ###################################################
 ## Make sure the mpi compilers can be found
 ###################################################
@@ -50,7 +55,7 @@ USER cam_sima_user
 WORKDIR /home/cam_sima_user/CAM-SIMA
 
 # pull the dependencies
-RUN ./manage_externals/checkout_externals
+RUN ./bin/git-fleximod update
 
 # Copy in the machine information for the container
 RUN cp /home/cam_sima_user/CAM-SIMA/docker/config_machines.xml /home/cam_sima_user/CAM-SIMA/ccs_config/machines/
@@ -79,7 +84,7 @@ RUN ./xmlchange STOP_N=5
 RUN chmod +x /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh
 RUN /home/cam_sima_user/CAM-SIMA/docker/ftp_download.sh
 
-# # add the snapshot file
+# add the snapshot file
 RUN echo "ncdata='/home/cam_sima_user/run_heldsuarez_cam6_nt2_bigg_try005.cam.h5.0001-01-01-00000.nc'" >> user_nl_cam
 
 RUN ./case.build

docker/Dockerfile

@@ -1,6 +1,6 @@
 # parts of CAM require x86 architecture (gptl, which relies on the rdtsc x86 assembly instruction)
 # esmf is am image you are expected to have built. Read the README file for instructions
-FROM esmf:latest
+FROM --platform=linux/amd64 esmf:latest
 
 ###################################################
 ## Install necessary packages
@@ -11,13 +11,18 @@ RUN dnf -y update \
       git \
       hostname \
       m4 \
-      python \
+      python39 \
+      pip \
       sudo \
       svn \
       tree \
       vim \
     && dnf clean all
 
+RUN ln -s $(which python3) /usr/bin/python && \
+    pip install --upgrade pip && \
+    pip install --upgrade setuptools
+
 ###################################################
 ## Make sure the mpi compilers can be found
 ###################################################
@@ -36,37 +41,22 @@ RUN cd pnetcdf-1.12.3 && \
 
 ENV FC=gfortran
 
-###################################################
-## Build and install json-fortran
-###################################################
-RUN curl -LO https://github.com/jacobwilliams/json-fortran/archive/8.2.0.tar.gz \
-    && tar -zxvf 8.2.0.tar.gz \
-    && cd json-fortran-8.2.0 \
-    && mkdir build \
-    && cd build \
-    && cmake -D SKIP_DOC_GEN:BOOL=TRUE .. \
-    && make install -j 8
-
-# add a symlink
-RUN ln -s /usr/local/jsonfortran-gnu-8.2.0/lib/libjsonfortran.a /usr/local/lib/libjsonfortran.a
-
 ###################################################
 ## Build and install MUSICA
 ###################################################
 
-RUN git clone https://github.com/NCAR/musica.git
+RUN git clone https://github.com/NCAR/musica.git \
+    && cd musica \
+    && git checkout 2a5eeaac982a3eb80b96d1e2087b91b301d1e748
+
 RUN mkdir /musica/build \
     && cd /musica/build \
-    && export JSON_FORTRAN_HOME="/usr/local/jsonfortran-gnu-8.2.0" \
     && cmake \
              -D ENABLE_TESTS=OFF \
-             -D ENABLE_TUVX=OFF \
-          .. \
+             -D MUSICA_BUILD_FORTRAN_INTERFACE=ON \
+             .. \
     && make install -j 8
 
-# add a symlink
-RUN ln -s /usr/local/musica-0.3.0/lib64/libmusica.a /usr/local/lib/libmusica.a
-
 ###################################################
 ## Build CAM-SIMA
 ###################################################
@@ -83,7 +73,7 @@ USER cam_sima_user
 WORKDIR /home/cam_sima_user/CAM-SIMA
 
 # pull the dependencies
-RUN ./manage_externals/checkout_externals
+RUN ./bin/git-fleximod update
 
 # Copy in the machine information for the container
 RUN cp /home/cam_sima_user/CAM-SIMA/docker/config_machines.xml /home/cam_sima_user/CAM-SIMA/ccs_config/machines/
@@ -104,7 +94,7 @@ WORKDIR $CASE_NAME
 RUN ./case.setup
 
 RUN ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites musica"
-RUN ./xmlchange CAM_LINKED_LIBS="-lmusica -ljsonfortran"
+RUN ./xmlchange CAM_LINKED_LIBS="-lmusica-fortran -lmusica -lyaml-cpp"
 RUN ./xmlchange ROF_NCPL=48
 RUN ./xmlchange STOP_OPTION=nsteps
 RUN ./xmlchange STOP_N=5

docker/Dockerfile.musica

@@ -561,7 +561,6 @@ subroutine cam_register_constituents(cam_runtime_opts)
       integer                                        :: errflg
       character(len=512)                             :: errmsg
       type(ccpp_constituent_prop_ptr_t), pointer     :: const_props(:)
-      type(ccpp_constituent_properties_t), allocatable, target :: dynamic_constituents(:)
       character(len=*), parameter :: subname = 'cam_register_constituents: '
 
       ! Initalize error flag and message: