CI simplifications

.github/workflows/ci.yml

@@ -90,18 +90,23 @@ jobs:
         include:
           - comp: gnu
             amrex_build_args: 'COMP=gnu'
+            amrex_build_flags: '--comp=gnu'
             dependency_cmds:
           - comp: llvm
             amrex_build_args: 'COMP=llvm'
+            amrex_build_flags: '--comp=llvm'
             dependency_cmds:
           - comp: cuda
             amrex_build_args: 'COMP=gnu USE_CUDA=TRUE'
+            amrex_build_flags: '--comp=gnu --with-cuda=yes'
             dependency_cmds: '.github/workflows/dependencies/dependencies_cuda.sh'
           - comp: hip
             amrex_build_args: 'USE_HIP=TRUE AMD_ARCH=gfx90a LIBRARY_LOCATIONS=/opt/rocm/lib'
+            amrex_build_flags: '--with-hip=yes'
             dependency_cmds: '.github/workflows/dependencies/dependencies_hip.sh'
           - comp: sycl
             amrex_build_args: 'USE_SYCL=TRUE'
+            amrex_build_flags: '--with-sycl=yes'
             dependency_cmds: '.github/workflows/dependencies/dependencies_dpcpp.sh'
     steps:
       - uses: actions/checkout@v4
@@ -151,7 +156,7 @@ jobs:
       - name: Dependencies
         working-directory: ${{env.TRANSPORT_WORKING_DIRECTORY}}
         run: |
-          ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh
+          printf "\n-------- Compiler Dependencies --------\n"; \
           if [ "${{matrix.comp}}" == 'cuda' ]; then \
             ${{github.workspace}}/PelePhysics-${{matrix.comp}}/${{matrix.dependency_cmds}}; \
             echo "CUDA_HOME=${CUDA_HOME}"; \
@@ -168,21 +173,22 @@ jobs:
             source /opt/intel/oneapi/setvars.sh || true
             which icpx || echo "icpx not in PATH!"; \
           fi
+          printf "\n-------- PelePhysics TPLs --------\n"; \
           cmake --version
           make TPL ${{matrix.amrex_build_args}};
-          if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
-            make TPL DEBUG=TRUE TINY_PROFILE=TRUE ${{matrix.amrex_build_args}};
-            cd ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/ceptr
-            poetry install
-          fi
+          printf "\n-------- AMReX --------\n"; \
+          cd ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Submodules/amrex/
+          ./configure --dim=2 --with-mpi=no --enable-particle=no --with-fortran=no --enable-linear-solver=no --enable-fortran-api=no --prefix=./ --enable-tiny-profile=yes ${{matrix.amrex_build_flags}}
+          make -j ${{env.NPROCS}} ${{matrix.amrex_build_args}} USE_SUNDIALS=TRUE SUNDIALS_HOME=${{github.workspace}}/PelePhysics-${{matrix.comp}}/ThirdParty/INSTALL/*/
+          make install
       - name: Unit Test Utilities
         working-directory: ${{env.UTILITIES_WORKING_DIRECTORY}}
         run: |
           echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json"
           if [ "${{matrix.comp}}" == 'hip' ]; then source /etc/profile.d/rocm.sh; fi;
           if [ "${{matrix.comp}}" == 'sycl' ]; then source /opt/intel/oneapi/setvars.sh || true; fi;
           ccache -z
-          make -j ${{env.NPROCS}} TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}; \
+          make -j ${{env.NPROCS}} TINY_PROFILE=TRUE USE_CCACHE=TRUE USE_AMREX_LIB=TRUE ${{matrix.amrex_build_args}}; \
           if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
             ./Pele2d.${{matrix.comp}}.TPROF.ex;
           fi;
@@ -205,7 +211,7 @@ jobs:
             if [ "${TYPE}" == 'Manifold' ]; then EOSMODEL=Manifold; else EOSMODEL=Fuego; fi; \
             if [ "${TYPE}" == 'Manifold' ]; then CHEMISTRY=Null; else CHEMISTRY=air; fi; \
             if [ "${TYPE}" != 'Manifold' ] || [ "${{matrix.comp}}" != 'sycl' ]; then \
-              make -j ${{env.NPROCS}} Eos_Model=${EOSMODEL} Chemistry_Model=${CHEMISTRY} Transport_Model=${TYPE} TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}; \
+              make -j ${{env.NPROCS}} Eos_Model=${EOSMODEL} Chemistry_Model=${CHEMISTRY} Transport_Model=${TYPE} TINY_PROFILE=TRUE USE_CCACHE=TRUE USE_AMREX_LIB=TRUE ${{matrix.amrex_build_args}}; \
             fi; \
             if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
               ./Pele2d.${{matrix.comp}}.TPROF.ex inputs.2d_${TYPE}; \
@@ -234,7 +240,7 @@ jobs:
             if [ "${TYPE}" == 'Manifold' ]; then INPUTFILE=inputs.2d.table2d; else INPUTFILE=inputs.2d; fi; \
             printf "\n-------- ${TYPE} --------\n"; \
             if [ "${TYPE}" != 'Manifold' ] || [ "${{matrix.comp}}" != 'sycl' ]; then \
-              make -j ${{env.NPROCS}} Eos_Model=${TYPE} Chemistry_Model=${CHEMISTRY} Transport_Model=${TRANSPORT} TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}; \
+              make -j ${{env.NPROCS}} Eos_Model=${TYPE} Chemistry_Model=${CHEMISTRY} Transport_Model=${TRANSPORT} TINY_PROFILE=TRUE USE_CCACHE=TRUE USE_AMREX_LIB=TRUE ${{matrix.amrex_build_args}}; \
             fi; \
             if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
               ./Pele2d.${{matrix.comp}}.TPROF.ex ${INPUTFILE}; \
@@ -257,22 +263,22 @@ jobs:
           if [ "${{matrix.comp}}" == 'hip' ]; then source /etc/profile.d/rocm.sh; fi;
           if [ "${{matrix.comp}}" == 'sycl' ]; then source /opt/intel/oneapi/setvars.sh || true; fi;
           ccache -z
-          make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=drm19 TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}
+          make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=drm19 TINY_PROFILE=TRUE USE_CCACHE=TRUE USE_AMREX_LIB=TRUE ${{matrix.amrex_build_args}}
           if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
             for TYPE in Cvode Arkode RK64; do \
               printf "\n-------- ${TYPE} --------\n"; \
-              ./Pele3d.${{matrix.comp}}.TPROF.ex inputs.3d_Array4 ode.dt=1.e-05 ode.ndt=100 chem_integrator="Reactor${TYPE}"; \
-              ./Pele3d.${{matrix.comp}}.TPROF.ex inputs.3d_1dArray ode.dt=1.e-05 ode.ndt=100 chem_integrator="Reactor${TYPE}"; \
+              ./Pele2d.${{matrix.comp}}.TPROF.ex inputs.3d_Array4 ode.dt=1.e-05 ode.ndt=100 chem_integrator="Reactor${TYPE}"; \
+              ./Pele2d.${{matrix.comp}}.TPROF.ex inputs.3d_1dArray ode.dt=1.e-05 ode.ndt=100 chem_integrator="Reactor${TYPE}"; \
               if [ $? -ne 0 ]; then exit 1; fi; \
             done \
           fi;
           make realclean
-          make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=LiDryer TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}
+          make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=LiDryer TINY_PROFILE=TRUE USE_CCACHE=TRUE USE_AMREX_LIB=TRUE ${{matrix.amrex_build_args}}
           if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
             for TYPE in Cvode Arkode RK64; do \
               printf "\n-------- ${TYPE} --------\n"; \
-               ./Pele3d.${{matrix.comp}}.TPROF.ex inputs.3d_Array4 fuel_name=H2 ode.dt=1.e-05 ode.ndt=100 chem_integrator="Reactor${TYPE}"; \
-               ./Pele3d.${{matrix.comp}}.TPROF.ex inputs.3d_1dArray fuel_name=H2 ode.dt=1.e-05 ode.ndt=100 chem_integrator="Reactor${TYPE}"; \
+               ./Pele2d.${{matrix.comp}}.TPROF.ex inputs.3d_Array4 fuel_name=H2 ode.dt=1.e-05 ode.ndt=100 chem_integrator="Reactor${TYPE}"; \
+               ./Pele2d.${{matrix.comp}}.TPROF.ex inputs.3d_1dArray fuel_name=H2 ode.dt=1.e-05 ode.ndt=100 chem_integrator="Reactor${TYPE}"; \
               if [ $? -ne 0 ]; then exit 1; fi; \
             done \
           fi
@@ -290,13 +296,13 @@ jobs:
               python -m pip install --upgrade pip
               pip install numpy
               ccache -z
-              make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}
+              make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu TINY_PROFILE=TRUE USE_CCACHE=TRUE USE_AMREX_LIB=TRUE ${{matrix.amrex_build_args}}
               bash exec_ignDelay.sh firstpass
               python check_ignDelay.py
               if [ $? -ne 0 ]; then exit 1; fi; \
               rm log PPreaction.txt inputs/inputs.0d_refine
               make realclean
-              make -j ${{env.NPROCS}} Eos_Model=Manifold Transport_Model=Manifold Manifold_Dim=1 Chemistry_Model=Null TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}
+              make -j ${{env.NPROCS}} Eos_Model=Manifold Transport_Model=Manifold Manifold_Dim=1 Chemistry_Model=Null TINY_PROFILE=TRUE USE_CCACHE=TRUE USE_AMREX_LIB=TRUE ${{matrix.amrex_build_args}}
               bash exec_ignDelay.sh manifold
               python check_ignDelay.py 0.0766 10
               if [ $? -ne 0 ]; then exit 1; fi; \
@@ -314,9 +320,11 @@ jobs:
           if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \
               ccache -z
               cd ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/ceptr
+              poetry install
               poetry run qssa -f ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Mechanisms/dodecane_lu_qss/skeletal.yaml -n ${{github.workspace}}/Mechanisms/dodecane_lu_qss/non_qssa_list.yaml
               poetry run convert -f ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Mechanisms/dodecane_lu_qss/qssa.yaml --qss_format_input ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Mechanisms/dodecane_lu_qss/qssa_input_non_optimized.toml --qss_symbolic_jacobian
               cd ${{env.JAC_WORKING_DIRECTORY}}
+              make TPL DEBUG=TRUE TINY_PROFILE=TRUE ${{matrix.amrex_build_args}}
               make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu_qss DEBUG=TRUE TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}
               ./Pele3d.${{matrix.comp}}.DEBUG.TPROF.ex; \
               if [ $? -ne 0 ]; then exit 1; fi; \

Testing/Exec/IgnitionDelay/GNUmakefile

@@ -1,5 +1,5 @@
 # AMReX
-DIM        = 3
+DIM        = 2
 PRECISION  = DOUBLE
 PROFILE    = FALSE
 VERBOSE    = FALSE

Testing/Exec/Make.PelePhysics

@@ -18,6 +18,10 @@ VERBOSE = TRUE
 EXTERN_CORE ?=
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
+ifeq ($(USE_AMREX_LIB), TRUE)
+  CEXE_sources :=
+endif
+
 
 include $(PELE_PHYSICS_HOME)/ThirdParty/Make.ThirdParty
 
@@ -132,10 +136,17 @@ include $(UTILITY_HOME)/TurbInflow/Make.package
 include $(UTILITY_HOME)/BlackBoxFunction/Make.package
 include $(UTILITY_HOME)/Utilities/Make.package
 
-Pdirs   := Base Boundary AmrCore
-
-Bpack	+= $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package)
-Blocs	+= $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir))
+ifeq ($(USE_AMREX_LIB), TRUE)
+  AMREX_LIBRARY_HOME ?= $(AMREX_HOME)
+  LIBDIR := $(AMREX_LIBRARY_HOME)/lib
+  INCDIR := $(AMREX_LIBRARY_HOME)/include
+  COMPILE_CPP_FLAGS ?= $(shell awk '/Cflags:/ {$$1=$$2=""; print $$0}' $(LIBDIR)/pkgconfig/amrex.pc)
+  COMPILE_LIB_FLAGS ?= $(shell awk '/Libs:/ {$$1=$$2=""; print $$0}' $(LIBDIR)/pkgconfig/amrex.pc)
+else
+  Pdirs := Base Boundary AmrCore
+  Bpack	+= $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package)
+  Blocs	+= $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir))
+endif
 
 include $(Bpack)
 
@@ -164,6 +175,11 @@ else
 	@echo SUCCESS
 endif
 
+ifeq ($(USE_AMREX_LIB), TRUE)
+  CXXFLAGS += -I$(INCDIR) $(COMPILE_CPP_FLAGS)
+  LIBRARIES += -L$(LIBDIR) $(COMPILE_LIB_FLAGS)
+endif
+
 include $(AMREX_HOME)/Tools/GNUMake/Make.rules
 
 ifeq ($(USE_CUDA),TRUE)

Testing/Exec/ReactEval/GNUmakefile

@@ -2,7 +2,7 @@
 PELE_PHYSICS_HOME    ?= ../../..
 
 # AMReX
-DIM          = 3
+DIM          = 2
 PRECISION    = DOUBLE
 PROFILE      = FALSE
 VERBOSE      = FALSE

Testing/Exec/ReactEval/GPU_misc.H

@@ -39,10 +39,10 @@ initialize_data(
   amrex::Real y = plo[1] + (j + 0.5) * dx[1];
   // amrex::Real x = plo[0] + (i+0.5)*dx[0];
   amrex::Real pi = 3.1415926535897932;
-  amrex::GpuArray<amrex::Real, 3> L;
-  amrex::GpuArray<amrex::Real, 3> P;
+  amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> L;
+  amrex::GpuArray<amrex::Real, AMREX_SPACEDIM> P;
 
-  for (int n = 0; n < 3; n++) {
+  for (int n = 0; n < AMREX_SPACEDIM; n++) {
     L[n] = phi[n] - plo[n];
     P[n] = L[n] / 4.0;
   }

Testing/Exec/ReactEval/inputs.3d_1dArray

@@ -7,7 +7,7 @@ chem_integrator = "ReactorRK64"
 
 ode.dt  = 2.e-05
 ode.ndt = 30
-ode.ode_ncells = 1024
+ode.ode_ncells = 128
 ode.reactor_type  = 1 # 1=full e, 2=full h
 ode.verbose = 1
 ode.use_typ_vals = 0