From a58f99c92d49e394be4d56690a0544a9eac1343f Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 25 Jul 2025 15:06:11 -0700 Subject: [PATCH 01/52] Created the QIHandler class for handling QuadraticInterpolator data, this will break the python interface and anything that uses QI so a WIP, created a QI test for GPUs, created the SPHERAL_GPU_ENABLED macro --- src/PYB11/Utilities/QuadraticInterpolator.py | 1 - src/Utilities/QuadraticInterpolator.cc | 96 +++++++++---------- src/Utilities/QuadraticInterpolator.hh | 87 +++++++++++------ src/Utilities/QuadraticInterpolatorInline.hh | 88 ++++++++++------- src/config.hh.in | 4 + tests/cpp/Utilities/CMakeLists.txt | 7 ++ .../Utilities/quadraticinterpolator_tests.cc | 94 ++++++++++++++++++ 7 files changed, 261 insertions(+), 116 deletions(-) create mode 100644 tests/cpp/Utilities/quadraticinterpolator_tests.cc diff --git a/src/PYB11/Utilities/QuadraticInterpolator.py b/src/PYB11/Utilities/QuadraticInterpolator.py index 64c25e830b..6d1a4450d3 100644 --- a/src/PYB11/Utilities/QuadraticInterpolator.py +++ b/src/PYB11/Utilities/QuadraticInterpolator.py @@ -96,4 +96,3 @@ def lowerBound(self, xmin = PYB11property(doc="Minimum x coordinate for table") xmax = PYB11property(doc="Maximum x coordinate for table") xstep = PYB11property(doc="delta x between tabulated values") - coeffs = PYB11property(doc="the fitting coefficients") diff --git a/src/Utilities/QuadraticInterpolator.cc b/src/Utilities/QuadraticInterpolator.cc index 7d4e6edfb6..c9728a71b0 100644 --- a/src/Utilities/QuadraticInterpolator.cc +++ b/src/Utilities/QuadraticInterpolator.cc @@ -7,51 +7,40 @@ // Created by JMO, Fri Dec 4 14:28:08 PST 2020 //----------------------------------------------------------------------------// #include "QuadraticInterpolator.hh" +#include "umpire/Umpire.hpp" +#include -namespace Spheral { +#include -//------------------------------------------------------------------------------ -// Default constructor -//------------------------------------------------------------------------------ -QuadraticInterpolator::QuadraticInterpolator(): - mN1(), - mXmin(), - mXmax(), - mXstep(), - mcoeffs() { -} +namespace Spheral { //------------------------------------------------------------------------------ // Constructor with sampled values //------------------------------------------------------------------------------ -QuadraticInterpolator::QuadraticInterpolator(double xmin, - double xmax, - const std::vector& yvals): - mN1(), - mXmin(), - mXmax(), - mXstep(), - mcoeffs() { - this->initialize(xmin, xmax, yvals); +QIHandler::QIHandler(double xmin, + double xmax, + const std::vector& yvals) { + initialize(xmin, xmax, yvals); } //------------------------------------------------------------------------------ // Initialize the interpolation to fit the given data //------------------------------------------------------------------------------ void -QuadraticInterpolator::initialize(double xmin, - double xmax, - const std::vector& yvals) { +QIHandler::initialize(double xmin, + double xmax, + const std::vector& yvals) { const auto n = yvals.size(); - VERIFY2(n > 2, "QuadraticInterpolator::initialize requires at least 3 unique values to fit"); - VERIFY2(n % 2 == 1, "QuadraticInterpolator::initialize requires an odd number of tabulated values"); - VERIFY2(xmax > xmin, "QuadraticInterpolator::initialize requires a positive domain: [" << xmin << " " << xmax << "]"); + VERIFY2(n > 2, "QIHandler::initialize requires at least 3 unique values to fit"); + VERIFY2(n % 2 == 1, "QIHandler::initialize requires an odd number of tabulated values"); + VERIFY2(xmax > xmin, "QIHandler::initialize requires a positive domain: [" << xmin << " " << xmax << "]"); - mN1 = (n - 1u)/2u - 1u; // Maximum index into arrays - mXmin = xmin; - mXmax = xmax; - mXstep = (xmax - xmin)/(mN1 + 1u); - mcoeffs.resize(3*(mN1 + 1u)); + double N1 = (n - 1u)/2u - 1u; // Maximum index into arrays + double xstep = (xmax - xmin)/(N1 + 1u); + size_t N = 3*(N1 + 1u); + auto& rm = umpire::ResourceManager::getInstance(); + auto host_allocator = rm.getAllocator("HOST"); + double* vals = static_cast(host_allocator.allocate(N*sizeof(double))); typedef Eigen::Matrix EMatrix; typedef Eigen::Matrix EVector; @@ -62,37 +51,42 @@ QuadraticInterpolator::initialize(double xmin, double x0, x1, x2; EMatrix A; EVector X, B; - for (auto i0 = 0u; i0 <= mN1; ++i0) { - x0 = xmin + i0*mXstep; - x1 = x0 + 0.5*mXstep; - x2 = x0 + mXstep; + for (auto i0 = 0u; i0 <= N1; ++i0) { + x0 = xmin + i0*xstep; + x1 = x0 + 0.5*xstep; + x2 = x0 + xstep; A << 1.0, x0, x0*x0, 1.0, x1, x1*x1, 1.0, x2, x2*x2; B << yvals[2u*i0], yvals[2u*i0 + 1u], yvals[2u*i0 + 2u]; X = A.inverse()*B; - mcoeffs[3*i0 ] = X(0); - mcoeffs[3*i0 + 1u] = X(1); - mcoeffs[3*i0 + 2u] = X(2); + vals[3*i0 ] = X(0); + vals[3*i0 + 1u] = X(1); + vals[3*i0 + 2u] = X(2); } +#ifdef SPHERAL_GPU_ENABLED + auto allocator = rm.getAllocator("DEVICE"); + mDeviceCoeffs = static_cast(allocator.allocate(N*sizeof(double))); + rm.copy(mDeviceCoeffs, vals, N*sizeof(double)); +#endif + mHostCoeffs = vals; + mN1 = N1; + mXmin = xmin; + mXmax = xmax; + mXstep = xstep; } //------------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------------ -QuadraticInterpolator::~QuadraticInterpolator() { -} - -//------------------------------------------------------------------------------ -// Equivalence -//------------------------------------------------------------------------------ -bool -QuadraticInterpolator:: -operator==(const QuadraticInterpolator& rhs) const { - return ((mN1 == rhs.mN1) and - (mXmin == rhs.mXmin) and - (mXmax == rhs.mXmax) and - (mcoeffs == rhs.mcoeffs)); +QIHandler::~QIHandler() { + auto& rm = umpire::ResourceManager::getInstance(); +#ifdef SPHERAL_GPU_ENABLED + auto allocator = rm.getAllocator("DEVICE"); + allocator.deallocate(mDeviceCoeffs); +#endif + auto host_allocator = rm.getAllocator("HOST"); + host_allocator.deallocate(mHostCoeffs); } } diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index 06a9fd169e..f85008e88f 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -9,6 +9,9 @@ #ifndef __Spheral_QuadraticInterpolator__ #define __Spheral_QuadraticInterpolator__ +#include "chai/ExecutionSpaces.hpp" +#include "config.hh" + #include #include @@ -18,48 +21,78 @@ class QuadraticInterpolator { public: //--------------------------- Public Interface ---------------------------// // Constructors, destructors - template - QuadraticInterpolator(double xmin, double xmax, size_t n, const Func& F); - QuadraticInterpolator(double xmin, double xmax, const std::vector& yvals); - QuadraticInterpolator(); - ~QuadraticInterpolator(); - - // Initialize after construction, either with a function or tabulated values - template - void initialize(double xmin, double xmax, size_t n, const Func& f); - void initialize(double xmin, double xmax, const std::vector& yvals); + SPHERAL_HOST_DEVICE QuadraticInterpolator() = default; + SPHERAL_HOST_DEVICE ~QuadraticInterpolator() = default; // Comparisons - bool operator==(const QuadraticInterpolator& rhs) const; + SPHERAL_HOST_DEVICE bool operator==(const QuadraticInterpolator& rhs) const; // Interpolate for the y value - double operator()(const double x) const; - double prime(const double x) const; // First derivative - double prime2(const double x) const; // Second derivative + SPHERAL_HOST_DEVICE double operator()(const double x) const; + SPHERAL_HOST_DEVICE double prime(const double x) const; // First derivative + SPHERAL_HOST_DEVICE double prime2(const double x) const; // Second derivative // Same as above, but use a pre-computed table position (from lowerBound) - double operator()(const double x, const size_t i0) const; - double prime(const double x, const size_t i0) const; // First derivative - double prime2(const double x, const size_t i0) const; // Second derivative + SPHERAL_HOST_DEVICE double operator()(const double x, const size_t i0) const; + SPHERAL_HOST_DEVICE double prime(const double x, const size_t i0) const; // First derivative + SPHERAL_HOST_DEVICE double prime2(const double x, const size_t i0) const; // Second derivative // Return the lower bound index in the table for the given x coordinate - size_t lowerBound(const double x) const; + SPHERAL_HOST_DEVICE size_t lowerBound(const double x) const; // Allow read access the internal data representation - size_t size() const; // The size of the tabulated coefficient arrays - double xmin() const; // Minimum x coordinate for table - double xmax() const; // Maximum x coordinate for table - double xstep() const; // delta x between tabulated values - const std::vector& coeffs() const; // the fitting coefficients - + SPHERAL_HOST_DEVICE size_t size() const; // The size of the tabulated coefficient arrays + SPHERAL_HOST_DEVICE double xmin() const; // Minimum x coordinate for table + SPHERAL_HOST_DEVICE double xmax() const; // Maximum x coordinate for table + SPHERAL_HOST_DEVICE double xstep() const; // delta x between tabulated values + private: //--------------------------- Private Interface --------------------------// + // Only QIHandler can to a proper construction of this object + SPHERAL_HOST QuadraticInterpolator(size_t N1, double xmin, double xmax, double xstep, double* vals); // Member data - size_t mN1; - double mXmin, mXmax, mXstep; - std::vector mcoeffs; + size_t mN1 = 0u; + double mXmin = 0.; + double mXmax = 0.; + double mXstep = 0.; + double* mcoeffs = nullptr; + friend class QIHandler; }; +class QIHandler { +public: + // Constructors, destructors + QIHandler() = default; + template + QIHandler(double xmin, double xmax, size_t n, const Func& F); + QIHandler(double xmin, double xmax, const std::vector& yvals); + ~QIHandler(); + + // Initialize after construction, either with a function or tabulated values + template + void initialize(double xmin, double xmax, size_t n, const Func& f); + void initialize(double xmin, double xmax, const std::vector& yvals); + + size_t size() const { return 3*(mN1 + 1u); } + using QI = QuadraticInterpolator; + QI view(chai::ExecutionSpace space) { + if (space == chai::CPU) { + return QI(mN1, mXmin, mXmax, mXstep, mHostCoeffs); + } else { + return QI(mN1, mXmin, mXmax, mXstep, mDeviceCoeffs); + } + } + +private: + //--------------------------- Private Interface --------------------------// + // Member data + size_t mN1 = 0u; + double mXmin = 0.; + double mXmax = 0.; + double mXstep = 0.; + double* mHostCoeffs = nullptr; + double* mDeviceCoeffs = nullptr; +}; } #include "QuadraticInterpolatorInline.hh" diff --git a/src/Utilities/QuadraticInterpolatorInline.hh b/src/Utilities/QuadraticInterpolatorInline.hh index ed007778d9..f03ad929a3 100644 --- a/src/Utilities/QuadraticInterpolatorInline.hh +++ b/src/Utilities/QuadraticInterpolatorInline.hh @@ -1,7 +1,4 @@ #include "Utilities/DBC.hh" -#include - -#include namespace Spheral { @@ -9,52 +6,61 @@ namespace Spheral { // Construct to fit the given function //------------------------------------------------------------------------------ template -inline -QuadraticInterpolator::QuadraticInterpolator(double xmin, - double xmax, - size_t n, - const Func& F): - mN1(), - mXmin(), - mXmax(), - mXstep(), - mcoeffs() { - this->initialize(xmin, xmax, n, F); +QIHandler::QIHandler(double xmin, + double xmax, + size_t n, + const Func& F) { + initialize(xmin, xmax, n, F); } //------------------------------------------------------------------------------ // Initialize to fit the given function //------------------------------------------------------------------------------ template -inline void -QuadraticInterpolator::initialize(double xmin, - double xmax, - size_t n, - const Func& F) { +QIHandler::initialize(double xmin, + double xmax, + size_t n, + const Func& F) { // Preconditions VERIFY2(n > 1, "QuadraticInterpolator requires n > 1 : n=" << n); VERIFY2(xmax > xmin, "QuadraticInterpolator requires a positive domain: [" << xmin << " " << xmax << "]"); // Build up an array of the function values and use the array based initialization. if (n % 2 == 0) ++n; // Need odd number of samples to hit both endpoints of the range - mXstep = (xmax - xmin)/(n - 1u); + double xstep = (xmax - xmin)/(n - 1u); std::vector yvals(n); - for (auto i = 0u; i < n; ++i) yvals[i] = F(xmin + i*mXstep); - this->initialize(xmin, xmax, yvals); + for (auto i = 0u; i < n; ++i) yvals[i] = F(xmin + i*xstep); + initialize(xmin, xmax, yvals); +} + +//------------------------------------------------------------------------------ +// Initialize QuadraticInterpolator +//------------------------------------------------------------------------------ +SPHERAL_HOST inline +QuadraticInterpolator::QuadraticInterpolator(size_t N1, + double xmin, + double xmax, + double xstep, + double* vals) : + mN1(N1), + mXmin(xmin), + mXmax(xmax), + mXstep(xstep), + mcoeffs(vals) { } //------------------------------------------------------------------------------ // Interpolate for the given x value. //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline double QuadraticInterpolator::operator()(const double x) const { const auto i0 = lowerBound(x); return mcoeffs[i0] + (mcoeffs[i0 + 1] + mcoeffs[i0 + 2]*x)*x; } -inline +SPHERAL_HOST_DEVICE inline double QuadraticInterpolator::operator()(const double x, const size_t i0) const { @@ -65,14 +71,14 @@ QuadraticInterpolator::operator()(const double x, //------------------------------------------------------------------------------ // Interpolate the first derivative the given x value. //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline double QuadraticInterpolator::prime(const double x) const { const auto i0 = lowerBound(x); return mcoeffs[i0 + 1] + 2.0*mcoeffs[i0 + 2]*x; } -inline +SPHERAL_HOST_DEVICE inline double QuadraticInterpolator::prime(const double x, const size_t i0) const { @@ -84,14 +90,14 @@ QuadraticInterpolator::prime(const double x, // Interpolate the second derivative for the given x value. // Just a constant value, so not a great fit. //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline double QuadraticInterpolator::prime2(const double x) const { const auto i0 = lowerBound(x); return 2.0*mcoeffs[i0 + 2]; } -inline +SPHERAL_HOST_DEVICE inline double QuadraticInterpolator::prime2(const double /*x*/, const size_t i0) const { @@ -102,7 +108,7 @@ QuadraticInterpolator::prime2(const double /*x*/, //------------------------------------------------------------------------------ // Return the lower bound entry in the table for the given x coordinate //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline size_t QuadraticInterpolator::lowerBound(const double x) const { const auto result = 3u*std::min(mN1, size_t(std::max(0.0, x - mXmin)/mXstep)); @@ -113,34 +119,42 @@ QuadraticInterpolator::lowerBound(const double x) const { //------------------------------------------------------------------------------ // Data accessors //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline size_t QuadraticInterpolator::size() const { - return mcoeffs.size(); + return 3*(mN1 + 1u); } -inline +SPHERAL_HOST_DEVICE inline double QuadraticInterpolator::xmin() const { return mXmin; } -inline +SPHERAL_HOST_DEVICE inline double QuadraticInterpolator::xmax() const { return mXmax; } -inline +SPHERAL_HOST_DEVICE inline double QuadraticInterpolator::xstep() const { return mXstep; } +//------------------------------------------------------------------------------ +// Equivalence +//------------------------------------------------------------------------------ +SPHERAL_HOST_DEVICE inline -const std::vector& -QuadraticInterpolator::coeffs() const { - return mcoeffs; +bool +QuadraticInterpolator:: +operator==(const QuadraticInterpolator& rhs) const { + return ((mN1 == rhs.mN1) and + (mXmin == rhs.mXmin) and + (mXmax == rhs.mXmax) and + (mcoeffs == rhs.mcoeffs)); } } diff --git a/src/config.hh.in b/src/config.hh.in index 8310b76d80..72ec623b44 100644 --- a/src/config.hh.in +++ b/src/config.hh.in @@ -13,6 +13,10 @@ #cmakedefine SPHERAL_ENABLE_HIP #cmakedefine SPHERAL_ENABLE_VVI +#if defined(SPHERAL_ENABLE_HIP) || defined(SPHERAL_ENABLE_CUDA) +#define SPHERAL_GPU_ENABLED +#endif + #if defined(SPHERAL_ENABLE_HIP) && defined(__HIP_DEVICE_COMPILE__) #define SPHERAL_GPU_ACTIVE #endif // SPHERAL_ENABLE_CUDA && __CUDACC__ diff --git a/tests/cpp/Utilities/CMakeLists.txt b/tests/cpp/Utilities/CMakeLists.txt index a3060609ae..119c30011c 100644 --- a/tests/cpp/Utilities/CMakeLists.txt +++ b/tests/cpp/Utilities/CMakeLists.txt @@ -3,3 +3,10 @@ spheral_add_test( SOURCES offload_tests.cc DEBUG_LINKER ) + +spheral_add_test( + NAME quadraticinterpolator_tests + SOURCES quadraticinterpolator_tests.cc + DEPENDS_ON Spheral_Utilities + #DEBUG_LINKER +) diff --git a/tests/cpp/Utilities/quadraticinterpolator_tests.cc b/tests/cpp/Utilities/quadraticinterpolator_tests.cc new file mode 100644 index 0000000000..931cc3d282 --- /dev/null +++ b/tests/cpp/Utilities/quadraticinterpolator_tests.cc @@ -0,0 +1,94 @@ +// Debug log printing can be quickly enabled for this unit test by uncommenting the +// definition below even if Spheral was not configured w/ SPHERAL_ENABLE_LOGGER=On. +// #define SPHERAL_ENABLE_LOGGER + +#include "chai/ExecutionSpaces.hpp" +#include "chai/Types.hpp" +#include "test-basic-exec-policies.hh" +#include "test-utilities.hh" + +#include "Utilities/QuadraticInterpolator.hh" +#include +#include + +using QI = Spheral::QuadraticInterpolator; + +class QuadraticInterpolatorTest : public ::testing::Test { +public: + const double xmin = 10.; + const double xmax = 100.; + SPHERAL_HOST_DEVICE static double func(const double x) { + return 1. + x*(2. + 3.*x); + } + std::vector makeVec(size_t N) { + double xstep = (xmax - xmin)/((double)N - 1.); + std::vector yvals(N); + for (size_t i = 0; i < N; ++i) { + double x = xmin + xstep*(double)i; + yvals[i] = func(x); + } + return yvals; + } +}; + +// Setting up G Test for FieldList +TYPED_TEST_SUITE_P(QuadraticInterpolatorTypedTest); +template class QuadraticInterpolatorTypedTest : public QuadraticInterpolatorTest {}; + +// Test multiple FieldLists holding the same Field +GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, FuncCtorTest) { + + const size_t NV = 41; + const double xmin = gpu_this->xmin; + const double xmax = gpu_this->xmax; + Spheral::QIHandler qih(xmin, xmax, NV, gpu_this->func); + chai::ExecutionSpace space = chai::CPU; + if (typeid(TypeParam) != typeid(RAJA::seq_exec)) { + space = chai::GPU; + } + size_t N = qih.size(); + QI qi = qih.view(space); + EXEC_IN_SPACE_BEGIN(TypeParam) + SPHERAL_ASSERT_EQ(qi.size(), N); + EXEC_IN_SPACE_END() + const double xstep = (xmax - xmin)/((double)NV - 1.); + RAJA::forall(TRS_UINT(0, NV), + [=] (size_t i) { + double x = xmin + xstep*(double)i; + double rval = gpu_this->func(x); + double ival = qi(x); + SPHERAL_ASSERT_FLOAT_EQ(rval, ival); + }); +} + +GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, VecCtorTest) { + + const size_t NV = 41; + std::vector yvals = gpu_this->makeVec(NV); + const double xmin = gpu_this->xmin; + const double xmax = gpu_this->xmax; + Spheral::QIHandler qih(xmin, xmax, yvals); + chai::ExecutionSpace space = chai::CPU; + if (typeid(TypeParam) != typeid(RAJA::seq_exec)) { + space = chai::GPU; + } + size_t N = qih.size(); + QI qi = qih.view(space); + EXEC_IN_SPACE_BEGIN(TypeParam) + SPHERAL_ASSERT_EQ(qi.size(), N); + EXEC_IN_SPACE_END() + const double xstep = (xmax - xmin)/((double)NV - 1.); + RAJA::forall(TRS_UINT(0, NV), + [=] (size_t i) { + double x = xmin + xstep*(double)i; + double rval = gpu_this->func(x); + double ival = qi(x); + SPHERAL_ASSERT_FLOAT_EQ(rval, ival); + }); +} + +REGISTER_TYPED_TEST_SUITE_P(QuadraticInterpolatorTypedTest, FuncCtorTest, + VecCtorTest); + +INSTANTIATE_TYPED_TEST_SUITE_P(QuadraticInterpolator, QuadraticInterpolatorTypedTest, + typename Spheral::Test::Types, ); From 86b7464ce3f18963eb25fdc3aaeccd9edbd2a8eb Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 28 Jul 2025 15:51:14 -0700 Subject: [PATCH 02/52] Created a QI base class, make the QI class the value class that inherits from base and can make a view instance, and create a QIView class that also inherits from base --- src/Utilities/QuadraticInterpolator.cc | 24 ++++---- src/Utilities/QuadraticInterpolator.hh | 57 +++++++++--------- src/Utilities/QuadraticInterpolatorInline.hh | 58 +++++++++---------- .../Utilities/quadraticinterpolator_tests.cc | 44 +++++++------- 4 files changed, 89 insertions(+), 94 deletions(-) diff --git a/src/Utilities/QuadraticInterpolator.cc b/src/Utilities/QuadraticInterpolator.cc index c9728a71b0..44e377377a 100644 --- a/src/Utilities/QuadraticInterpolator.cc +++ b/src/Utilities/QuadraticInterpolator.cc @@ -17,9 +17,9 @@ namespace Spheral { //------------------------------------------------------------------------------ // Constructor with sampled values //------------------------------------------------------------------------------ -QIHandler::QIHandler(double xmin, - double xmax, - const std::vector& yvals) { +QuadraticInterpolator::QuadraticInterpolator(double xmin, + double xmax, + const std::vector& yvals) { initialize(xmin, xmax, yvals); } @@ -27,13 +27,13 @@ QIHandler::QIHandler(double xmin, // Initialize the interpolation to fit the given data //------------------------------------------------------------------------------ void -QIHandler::initialize(double xmin, - double xmax, - const std::vector& yvals) { +QuadraticInterpolator::initialize(double xmin, + double xmax, + const std::vector& yvals) { const auto n = yvals.size(); - VERIFY2(n > 2, "QIHandler::initialize requires at least 3 unique values to fit"); - VERIFY2(n % 2 == 1, "QIHandler::initialize requires an odd number of tabulated values"); - VERIFY2(xmax > xmin, "QIHandler::initialize requires a positive domain: [" << xmin << " " << xmax << "]"); + VERIFY2(n > 2, "QuadraticInterpolator::initialize requires at least 3 unique values to fit"); + VERIFY2(n % 2 == 1, "QuadraticInterpolator::initialize requires an odd number of tabulated values"); + VERIFY2(xmax > xmin, "QuadraticInterpolator::initialize requires a positive domain: [" << xmin << " " << xmax << "]"); double N1 = (n - 1u)/2u - 1u; // Maximum index into arrays double xstep = (xmax - xmin)/(N1 + 1u); @@ -69,7 +69,7 @@ QIHandler::initialize(double xmin, mDeviceCoeffs = static_cast(allocator.allocate(N*sizeof(double))); rm.copy(mDeviceCoeffs, vals, N*sizeof(double)); #endif - mHostCoeffs = vals; + mcoeffs = vals; mN1 = N1; mXmin = xmin; mXmax = xmax; @@ -79,14 +79,14 @@ QIHandler::initialize(double xmin, //------------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------------ -QIHandler::~QIHandler() { +QuadraticInterpolator::~QuadraticInterpolator() { auto& rm = umpire::ResourceManager::getInstance(); #ifdef SPHERAL_GPU_ENABLED auto allocator = rm.getAllocator("DEVICE"); allocator.deallocate(mDeviceCoeffs); #endif auto host_allocator = rm.getAllocator("HOST"); - host_allocator.deallocate(mHostCoeffs); + host_allocator.deallocate(mcoeffs); } } diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index f85008e88f..8e2d337266 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -17,15 +17,14 @@ namespace Spheral { -class QuadraticInterpolator { +class QuadraticInterpolatorBase { public: //--------------------------- Public Interface ---------------------------// // Constructors, destructors - SPHERAL_HOST_DEVICE QuadraticInterpolator() = default; - SPHERAL_HOST_DEVICE ~QuadraticInterpolator() = default; + SPHERAL_HOST_DEVICE QuadraticInterpolatorBase() = default; // Comparisons - SPHERAL_HOST_DEVICE bool operator==(const QuadraticInterpolator& rhs) const; + SPHERAL_HOST_DEVICE bool operator==(const QuadraticInterpolatorBase& rhs) const; // Interpolate for the y value SPHERAL_HOST_DEVICE double operator()(const double x) const; @@ -46,53 +45,57 @@ public: SPHERAL_HOST_DEVICE double xmax() const; // Maximum x coordinate for table SPHERAL_HOST_DEVICE double xstep() const; // delta x between tabulated values -private: +protected: + SPHERAL_HOST QuadraticInterpolatorBase(size_t N1, + double xmin, + double xmax, + double xstep, + double* vals) : + mN1(N1), + mXmin(xmin), + mXmax(xmax), + mXstep(xstep), + mcoeffs(vals) { } //--------------------------- Private Interface --------------------------// - // Only QIHandler can to a proper construction of this object - SPHERAL_HOST QuadraticInterpolator(size_t N1, double xmin, double xmax, double xstep, double* vals); // Member data size_t mN1 = 0u; double mXmin = 0.; double mXmax = 0.; double mXstep = 0.; double* mcoeffs = nullptr; - friend class QIHandler; }; -class QIHandler { +class QuadraticInterpolator : public QuadraticInterpolatorBase { public: - // Constructors, destructors - QIHandler() = default; template - QIHandler(double xmin, double xmax, size_t n, const Func& F); - QIHandler(double xmin, double xmax, const std::vector& yvals); - ~QIHandler(); + QuadraticInterpolator(double xmin, double xmax, size_t n, const Func& F); + QuadraticInterpolator(double xmin, double xmax, const std::vector& yvals); + ~QuadraticInterpolator(); // Initialize after construction, either with a function or tabulated values template void initialize(double xmin, double xmax, size_t n, const Func& f); void initialize(double xmin, double xmax, const std::vector& yvals); - size_t size() const { return 3*(mN1 + 1u); } - using QI = QuadraticInterpolator; - QI view(chai::ExecutionSpace space) { + template + QIView view(chai::ExecutionSpace space) { if (space == chai::CPU) { - return QI(mN1, mXmin, mXmax, mXstep, mHostCoeffs); + return QIView(mN1, mXmin, mXmax, mXstep, mcoeffs); } else { - return QI(mN1, mXmin, mXmax, mXstep, mDeviceCoeffs); + return QIView(mN1, mXmin, mXmax, mXstep, mDeviceCoeffs); } } - private: - //--------------------------- Private Interface --------------------------// - // Member data - size_t mN1 = 0u; - double mXmin = 0.; - double mXmax = 0.; - double mXstep = 0.; - double* mHostCoeffs = nullptr; double* mDeviceCoeffs = nullptr; }; + +// For use on device +class QIView : public QuadraticInterpolatorBase { +public: + SPHERAL_HOST_DEVICE QIView() = default; + SPHERAL_HOST QIView(size_t N1, double xmin, double xmax, double xstep, double* vals); + SPHERAL_HOST_DEVICE ~QIView() { } +}; } #include "QuadraticInterpolatorInline.hh" diff --git a/src/Utilities/QuadraticInterpolatorInline.hh b/src/Utilities/QuadraticInterpolatorInline.hh index f03ad929a3..5b8439d538 100644 --- a/src/Utilities/QuadraticInterpolatorInline.hh +++ b/src/Utilities/QuadraticInterpolatorInline.hh @@ -6,10 +6,10 @@ namespace Spheral { // Construct to fit the given function //------------------------------------------------------------------------------ template -QIHandler::QIHandler(double xmin, - double xmax, - size_t n, - const Func& F) { +QuadraticInterpolator::QuadraticInterpolator(double xmin, + double xmax, + size_t n, + const Func& F) { initialize(xmin, xmax, n, F); } @@ -18,10 +18,10 @@ QIHandler::QIHandler(double xmin, //------------------------------------------------------------------------------ template void -QIHandler::initialize(double xmin, - double xmax, - size_t n, - const Func& F) { +QuadraticInterpolator::initialize(double xmin, + double xmax, + size_t n, + const Func& F) { // Preconditions VERIFY2(n > 1, "QuadraticInterpolator requires n > 1 : n=" << n); VERIFY2(xmax > xmin, "QuadraticInterpolator requires a positive domain: [" << xmin << " " << xmax << "]"); @@ -38,16 +38,12 @@ QIHandler::initialize(double xmin, // Initialize QuadraticInterpolator //------------------------------------------------------------------------------ SPHERAL_HOST inline -QuadraticInterpolator::QuadraticInterpolator(size_t N1, - double xmin, - double xmax, - double xstep, - double* vals) : - mN1(N1), - mXmin(xmin), - mXmax(xmax), - mXstep(xstep), - mcoeffs(vals) { +QIView::QIView(size_t N1, + double xmin, + double xmax, + double xstep, + double* vals) : + QuadraticInterpolatorBase(N1, xmin, xmax, xstep, vals) { } //------------------------------------------------------------------------------ @@ -55,14 +51,14 @@ QuadraticInterpolator::QuadraticInterpolator(size_t N1, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -QuadraticInterpolator::operator()(const double x) const { +QuadraticInterpolatorBase::operator()(const double x) const { const auto i0 = lowerBound(x); return mcoeffs[i0] + (mcoeffs[i0 + 1] + mcoeffs[i0 + 2]*x)*x; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolator::operator()(const double x, +QuadraticInterpolatorBase::operator()(const double x, const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return mcoeffs[i0] + (mcoeffs[i0 + 1] + mcoeffs[i0 + 2]*x)*x; @@ -73,14 +69,14 @@ QuadraticInterpolator::operator()(const double x, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -QuadraticInterpolator::prime(const double x) const { +QuadraticInterpolatorBase::prime(const double x) const { const auto i0 = lowerBound(x); return mcoeffs[i0 + 1] + 2.0*mcoeffs[i0 + 2]*x; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolator::prime(const double x, +QuadraticInterpolatorBase::prime(const double x, const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return mcoeffs[i0 + 1] + 2.0*mcoeffs[i0 + 2]*x; @@ -92,14 +88,14 @@ QuadraticInterpolator::prime(const double x, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -QuadraticInterpolator::prime2(const double x) const { +QuadraticInterpolatorBase::prime2(const double x) const { const auto i0 = lowerBound(x); return 2.0*mcoeffs[i0 + 2]; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolator::prime2(const double /*x*/, +QuadraticInterpolatorBase::prime2(const double /*x*/, const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return 2.0*mcoeffs[i0 + 2]; @@ -110,7 +106,7 @@ QuadraticInterpolator::prime2(const double /*x*/, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline size_t -QuadraticInterpolator::lowerBound(const double x) const { +QuadraticInterpolatorBase::lowerBound(const double x) const { const auto result = 3u*std::min(mN1, size_t(std::max(0.0, x - mXmin)/mXstep)); ENSURE(result <= 3u*mN1); return result; @@ -121,25 +117,25 @@ QuadraticInterpolator::lowerBound(const double x) const { //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline size_t -QuadraticInterpolator::size() const { +QuadraticInterpolatorBase::size() const { return 3*(mN1 + 1u); } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolator::xmin() const { +QuadraticInterpolatorBase::xmin() const { return mXmin; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolator::xmax() const { +QuadraticInterpolatorBase::xmax() const { return mXmax; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolator::xstep() const { +QuadraticInterpolatorBase::xstep() const { return mXstep; } @@ -149,8 +145,8 @@ QuadraticInterpolator::xstep() const { SPHERAL_HOST_DEVICE inline bool -QuadraticInterpolator:: -operator==(const QuadraticInterpolator& rhs) const { +QuadraticInterpolatorBase:: +operator==(const QuadraticInterpolatorBase& rhs) const { return ((mN1 == rhs.mN1) and (mXmin == rhs.mXmin) and (mXmax == rhs.mXmax) and diff --git a/tests/cpp/Utilities/quadraticinterpolator_tests.cc b/tests/cpp/Utilities/quadraticinterpolator_tests.cc index 931cc3d282..914c6fa023 100644 --- a/tests/cpp/Utilities/quadraticinterpolator_tests.cc +++ b/tests/cpp/Utilities/quadraticinterpolator_tests.cc @@ -37,43 +37,39 @@ template class QuadraticInterpolatorTypedTest : public QuadraticInt // Test multiple FieldLists holding the same Field GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, FuncCtorTest) { - const size_t NV = 41; const double xmin = gpu_this->xmin; const double xmax = gpu_this->xmax; - Spheral::QIHandler qih(xmin, xmax, NV, gpu_this->func); - chai::ExecutionSpace space = chai::CPU; - if (typeid(TypeParam) != typeid(RAJA::seq_exec)) { - space = chai::GPU; + QI qih(xmin, xmax, NV, gpu_this->func); + { + chai::ExecutionSpace space = chai::CPU; + if (typeid(TypeParam) != typeid(RAJA::seq_exec)) space = chai::GPU; + size_t N = qih.size(); + Spheral::QIView qi = qih.view(space); + EXEC_IN_SPACE_BEGIN(TypeParam) + SPHERAL_ASSERT_EQ(qi.size(), N); + EXEC_IN_SPACE_END() + const double xstep = (xmax - xmin)/((double)NV - 1.); + RAJA::forall(TRS_UINT(0, NV), + [=] (size_t i) { + double x = xmin + xstep*(double)i; + double rval = gpu_this->func(x); + double ival = qi(x); + SPHERAL_ASSERT_FLOAT_EQ(rval, ival); + }); } - size_t N = qih.size(); - QI qi = qih.view(space); - EXEC_IN_SPACE_BEGIN(TypeParam) - SPHERAL_ASSERT_EQ(qi.size(), N); - EXEC_IN_SPACE_END() - const double xstep = (xmax - xmin)/((double)NV - 1.); - RAJA::forall(TRS_UINT(0, NV), - [=] (size_t i) { - double x = xmin + xstep*(double)i; - double rval = gpu_this->func(x); - double ival = qi(x); - SPHERAL_ASSERT_FLOAT_EQ(rval, ival); - }); } GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, VecCtorTest) { - const size_t NV = 41; std::vector yvals = gpu_this->makeVec(NV); const double xmin = gpu_this->xmin; const double xmax = gpu_this->xmax; - Spheral::QIHandler qih(xmin, xmax, yvals); + QI qih(xmin, xmax, yvals); chai::ExecutionSpace space = chai::CPU; - if (typeid(TypeParam) != typeid(RAJA::seq_exec)) { - space = chai::GPU; - } + if (typeid(TypeParam) != typeid(RAJA::seq_exec)) space = chai::GPU; size_t N = qih.size(); - QI qi = qih.view(space); + Spheral::QIView qi = qih.view(space); EXEC_IN_SPACE_BEGIN(TypeParam) SPHERAL_ASSERT_EQ(qi.size(), N); EXEC_IN_SPACE_END() From 23d8ba22e43ecfba9d98afbe30d710d08204e8be Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 28 Jul 2025 16:50:28 -0700 Subject: [PATCH 03/52] Switched QI to use managed arrays instead of calling Umpire directly --- src/Utilities/QuadraticInterpolator.cc | 27 +++++--------------- src/Utilities/QuadraticInterpolator.hh | 21 +++++++-------- src/Utilities/QuadraticInterpolatorInline.hh | 2 +- 3 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/Utilities/QuadraticInterpolator.cc b/src/Utilities/QuadraticInterpolator.cc index 44e377377a..0e46aca6df 100644 --- a/src/Utilities/QuadraticInterpolator.cc +++ b/src/Utilities/QuadraticInterpolator.cc @@ -7,7 +7,6 @@ // Created by JMO, Fri Dec 4 14:28:08 PST 2020 //----------------------------------------------------------------------------// #include "QuadraticInterpolator.hh" -#include "umpire/Umpire.hpp" #include #include @@ -35,12 +34,10 @@ QuadraticInterpolator::initialize(double xmin, VERIFY2(n % 2 == 1, "QuadraticInterpolator::initialize requires an odd number of tabulated values"); VERIFY2(xmax > xmin, "QuadraticInterpolator::initialize requires a positive domain: [" << xmin << " " << xmax << "]"); - double N1 = (n - 1u)/2u - 1u; // Maximum index into arrays + size_t N1 = (n - 1u)/2u - 1u; // Maximum index into arrays double xstep = (xmax - xmin)/(N1 + 1u); size_t N = 3*(N1 + 1u); - auto& rm = umpire::ResourceManager::getInstance(); - auto host_allocator = rm.getAllocator("HOST"); - double* vals = static_cast(host_allocator.allocate(N*sizeof(double))); + mcoeffs.allocate(N, chai::CPU); typedef Eigen::Matrix EMatrix; typedef Eigen::Matrix EVector; @@ -60,16 +57,10 @@ QuadraticInterpolator::initialize(double xmin, 1.0, x2, x2*x2; B << yvals[2u*i0], yvals[2u*i0 + 1u], yvals[2u*i0 + 2u]; X = A.inverse()*B; - vals[3*i0 ] = X(0); - vals[3*i0 + 1u] = X(1); - vals[3*i0 + 2u] = X(2); + mcoeffs[3*i0 ] = X(0); + mcoeffs[3*i0 + 1u] = X(1); + mcoeffs[3*i0 + 2u] = X(2); } -#ifdef SPHERAL_GPU_ENABLED - auto allocator = rm.getAllocator("DEVICE"); - mDeviceCoeffs = static_cast(allocator.allocate(N*sizeof(double))); - rm.copy(mDeviceCoeffs, vals, N*sizeof(double)); -#endif - mcoeffs = vals; mN1 = N1; mXmin = xmin; mXmax = xmax; @@ -80,13 +71,7 @@ QuadraticInterpolator::initialize(double xmin, // Destructor //------------------------------------------------------------------------------ QuadraticInterpolator::~QuadraticInterpolator() { - auto& rm = umpire::ResourceManager::getInstance(); -#ifdef SPHERAL_GPU_ENABLED - auto allocator = rm.getAllocator("DEVICE"); - allocator.deallocate(mDeviceCoeffs); -#endif - auto host_allocator = rm.getAllocator("HOST"); - host_allocator.deallocate(mcoeffs); + mcoeffs.free(); } } diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index 8e2d337266..078543acb1 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -9,7 +9,7 @@ #ifndef __Spheral_QuadraticInterpolator__ #define __Spheral_QuadraticInterpolator__ -#include "chai/ExecutionSpaces.hpp" +#include "chai/ManagedArray.hpp" #include "config.hh" #include @@ -19,6 +19,7 @@ namespace Spheral { class QuadraticInterpolatorBase { public: + using ContainerType = typename chai::ManagedArray; //--------------------------- Public Interface ---------------------------// // Constructors, destructors SPHERAL_HOST_DEVICE QuadraticInterpolatorBase() = default; @@ -50,23 +51,24 @@ protected: double xmin, double xmax, double xstep, - double* vals) : + ContainerType const& vals) : mN1(N1), mXmin(xmin), mXmax(xmax), mXstep(xstep), - mcoeffs(vals) { } + mcoeffs(vals) { mcoeffs.registerTouch(chai::CPU); } //--------------------------- Private Interface --------------------------// // Member data size_t mN1 = 0u; double mXmin = 0.; double mXmax = 0.; double mXstep = 0.; - double* mcoeffs = nullptr; + ContainerType mcoeffs; }; class QuadraticInterpolator : public QuadraticInterpolatorBase { public: + using ContainerType = typename chai::ManagedArray; template QuadraticInterpolator(double xmin, double xmax, size_t n, const Func& F); QuadraticInterpolator(double xmin, double xmax, const std::vector& yvals); @@ -79,21 +81,16 @@ public: template QIView view(chai::ExecutionSpace space) { - if (space == chai::CPU) { - return QIView(mN1, mXmin, mXmax, mXstep, mcoeffs); - } else { - return QIView(mN1, mXmin, mXmax, mXstep, mDeviceCoeffs); - } + return QIView(mN1, mXmin, mXmax, mXstep, mcoeffs); } -private: - double* mDeviceCoeffs = nullptr; }; // For use on device class QIView : public QuadraticInterpolatorBase { public: + using ContainerType = typename chai::ManagedArray; SPHERAL_HOST_DEVICE QIView() = default; - SPHERAL_HOST QIView(size_t N1, double xmin, double xmax, double xstep, double* vals); + SPHERAL_HOST QIView(size_t N1, double xmin, double xmax, double xstep, ContainerType const& vals); SPHERAL_HOST_DEVICE ~QIView() { } }; } diff --git a/src/Utilities/QuadraticInterpolatorInline.hh b/src/Utilities/QuadraticInterpolatorInline.hh index 5b8439d538..081a01e329 100644 --- a/src/Utilities/QuadraticInterpolatorInline.hh +++ b/src/Utilities/QuadraticInterpolatorInline.hh @@ -42,7 +42,7 @@ QIView::QIView(size_t N1, double xmin, double xmax, double xstep, - double* vals) : + chai::ManagedArray const& vals) : QuadraticInterpolatorBase(N1, xmin, xmax, xstep, vals) { } From b6aa3250f821e2cf45257137a7f50d2fbb3fdfdd Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 28 Jul 2025 19:08:17 -0700 Subject: [PATCH 04/52] Removed the QIView class, GPUified the CubicHermitInterpolator and added tests --- src/Utilities/CubicHermiteInterpolator.cc | 63 ++--------- src/Utilities/CubicHermiteInterpolator.hh | 107 +++++++++++------- .../CubicHermiteInterpolatorInline.hh | 90 ++++++--------- src/Utilities/QuadraticInterpolator.hh | 34 ++---- src/Utilities/QuadraticInterpolatorInline.hh | 42 +++---- tests/cpp/Utilities/CMakeLists.txt | 7 ++ tests/cpp/Utilities/chinterpolator_tests.cc | 85 ++++++++++++++ .../Utilities/quadraticinterpolator_tests.cc | 10 +- 8 files changed, 238 insertions(+), 200 deletions(-) create mode 100644 tests/cpp/Utilities/chinterpolator_tests.cc diff --git a/src/Utilities/CubicHermiteInterpolator.cc b/src/Utilities/CubicHermiteInterpolator.cc index 654f0a0b20..47e55f72f6 100644 --- a/src/Utilities/CubicHermiteInterpolator.cc +++ b/src/Utilities/CubicHermiteInterpolator.cc @@ -22,50 +22,8 @@ namespace Spheral { CubicHermiteInterpolator:: CubicHermiteInterpolator(const double xmin, const double xmax, - const std::vector& yvals): - mN(), - mXmin(), - mXmax(), - mXstep(), - mVals() { - this->initialize(xmin, xmax, yvals); -} - -//------------------------------------------------------------------------------ -// Default constructor -//------------------------------------------------------------------------------ -CubicHermiteInterpolator::CubicHermiteInterpolator(): - mN(), - mXmin(), - mXmax(), - mXstep(), - mVals() { -} - -//------------------------------------------------------------------------------ -// Copy constructor -//------------------------------------------------------------------------------ -CubicHermiteInterpolator::CubicHermiteInterpolator(const CubicHermiteInterpolator& rhs): - mN(rhs.mN), - mXmin(rhs.mXmin), - mXmax(rhs.mXmax), - mXstep(rhs.mXstep), - mVals(rhs.mVals) { -} - -//------------------------------------------------------------------------------ -// Assignment -//------------------------------------------------------------------------------ -CubicHermiteInterpolator& -CubicHermiteInterpolator::operator=(const CubicHermiteInterpolator& rhs) { - if (this != &rhs) { - mN = rhs.mN; - mXmin = rhs.mXmin; - mXmax = rhs.mXmax; - mXstep = rhs.mXstep; - mVals = rhs.mVals; - } - return *this; + const std::vector& yvals) { + initialize(xmin, xmax, yvals); } //------------------------------------------------------------------------------ @@ -82,13 +40,13 @@ CubicHermiteInterpolator::initialize(const double xmin, mXmin = xmin; mXmax = xmax; mXstep = (xmax - xmin)/(mN - 1u); - mVals.resize(2u*mN); - - // Copy the function values - std::copy(yvals.begin(), yvals.end(), mVals.begin()); + mVals.allocate(2u*mN, chai::CPU); + for (size_t i = 0; i < mN; ++i) mVals[i] = yvals[i]; // Estimate the gradients at our lattice points - this->initializeGradientKnots(); + initializeGradientKnots(); + + // const auto dxInv = 1.0/mXstep; // for (auto i = 1u; i < mN - 1u; ++i) { // mVals[mN + i] = 0.5*(mVals[i + 1u] - mVals[i - 1u])*dxInv; @@ -101,14 +59,15 @@ CubicHermiteInterpolator::initialize(const double xmin, // Destructor //------------------------------------------------------------------------------ CubicHermiteInterpolator::~CubicHermiteInterpolator() { + mVals.free(); } //------------------------------------------------------------------------------ // Equivalence //------------------------------------------------------------------------------ -bool -CubicHermiteInterpolator:: -operator==(const CubicHermiteInterpolator& rhs) const { +SPHERAL_HOST_DEVICE bool +CHIBase:: +operator==(const CHIBase& rhs) const { return ((mN == rhs.mN) and (mXmin == rhs.mXmin) and (mXmax == rhs.mXmax) and diff --git a/src/Utilities/CubicHermiteInterpolator.hh b/src/Utilities/CubicHermiteInterpolator.hh index c2e2d04ad6..8e11164893 100644 --- a/src/Utilities/CubicHermiteInterpolator.hh +++ b/src/Utilities/CubicHermiteInterpolator.hh @@ -8,12 +8,68 @@ #ifndef __Spheral_CubicHermiteInterpolator__ #define __Spheral_CubicHermiteInterpolator__ +#include "chai/ManagedArray.hpp" +#include "config.hh" + #include #include namespace Spheral { +class CHIBase { +public: + using ContainerType = typename chai::ManagedArray; + //--------------------------- Public Interface ---------------------------// + // Constructors, destructors + SPHERAL_HOST_DEVICE CHIBase() = default; + // Comparisons + SPHERAL_HOST_DEVICE bool operator==(const CHIBase& rhs) const; + + // Interpolate for the y value + SPHERAL_HOST_DEVICE double operator()(const double x) const; + SPHERAL_HOST_DEVICE double prime(const double x) const; // First derivative + SPHERAL_HOST_DEVICE double prime2(const double x) const; // Second derivative + + // Same as above, but use a pre-computed table position (from lowerBound) + SPHERAL_HOST_DEVICE double operator()(const double x, const size_t i0) const; + SPHERAL_HOST_DEVICE double prime(const double x, const size_t i0) const; // First derivative + SPHERAL_HOST_DEVICE double prime2(const double x, const size_t i0) const; // Second derivative + + // Return the lower bound index in the table for the given x coordinate + SPHERAL_HOST_DEVICE size_t lowerBound(const double x) const; + + // Compute the Hermite basis functions + SPHERAL_HOST_DEVICE double h00(const double x) const; + SPHERAL_HOST_DEVICE double h10(const double x) const; + SPHERAL_HOST_DEVICE double h01(const double x) const; + SPHERAL_HOST_DEVICE double h11(const double x) const; -class CubicHermiteInterpolator { + // Allow read access the internal data representation + SPHERAL_HOST_DEVICE size_t size() const; // The number of tabulated values + SPHERAL_HOST_DEVICE double xmin() const; // Minimum x coordinate for table + SPHERAL_HOST_DEVICE double xmax() const; // Maximum x coordinate for table + SPHERAL_HOST_DEVICE double xstep() const; // delta x between tabulated values + + SPHERAL_HOST CHIBase(size_t N, + double xmin, + double xmax, + double xstep, + ContainerType const& vals) : + mN(N), + mXmin(xmin), + mXmax(xmax), + mXstep(xstep), + mVals(vals) { mVals.registerTouch(chai::CPU); } +protected: + //--------------------------- Protected Interface --------------------------// + // Member data + size_t mN = 0u; + double mXmin = 0.; + double mXmax = 0.; + double mXstep = 0.; + ContainerType mVals; +}; + +class CubicHermiteInterpolator : public CHIBase { public: //--------------------------- Public Interface ---------------------------// // Constructors, destructors @@ -34,10 +90,6 @@ public: CubicHermiteInterpolator(); ~CubicHermiteInterpolator(); - // Copy and assignment - CubicHermiteInterpolator(const CubicHermiteInterpolator& rhs); - CubicHermiteInterpolator& operator=(const CubicHermiteInterpolator& rhs); - // (Re)initialize after construction, same options as construction template void initialize(const double xmin, @@ -57,49 +109,26 @@ public: // Force interpolation to be monotonic (may introduce structure between tabulated points) void makeMonotonic(); - // Comparisons - bool operator==(const CubicHermiteInterpolator& rhs) const; - - // Interpolate for the y value - double operator()(const double x) const; - double prime(const double x) const; // First derivative - double prime2(const double x) const; // Second derivative - - // Same as above, but use a pre-computed table position (from lowerBound) - double operator()(const double x, const size_t i0) const; - double prime(const double x, const size_t i0) const; // First derivative - double prime2(const double x, const size_t i0) const; // Second derivative + CHIBase view() { + return CHIBase(mN, mXmin, mXmax, mXstep, mVals); + } - // Return the lower bound index in the table for the given x coordinate - size_t lowerBound(const double x) const; - - // Compute the Hermite basis functions - double h00(const double x) const; - double h10(const double x) const; - double h01(const double x) const; - double h11(const double x) const; - - // Allow read access the internal data representation - size_t size() const; // The number of tabulated values - double xmin() const; // Minimum x coordinate for table - double xmax() const; // Maximum x coordinate for table - double xstep() const; // delta x between tabulated values - const std::vector& vals() const; // the tabulated function values and gradients (sequentially) - private: //--------------------------- Private Interface --------------------------// - // Member data - size_t mN; - double mXmin, mXmax, mXstep; - std::vector mVals; - // Initialize the gradient at the interpolation points based on the tabulated // interpolation values void initializeGradientKnots(); }; +// For use on device +// class CHIView : public CHIBase { +// public: +// using ContainerType = typename chai::ManagedArray; +// SPHERAL_HOST_DEVICE CHIView() = default; +// SPHERAL_HOST CHIView(size_t N1, double xmin, double xmax, double xstep, ContainerType const& vals); +// SPHERAL_HOST_DEVICE ~CHIView() { } +// }; } - #include "CubicHermiteInterpolatorInline.hh" #endif diff --git a/src/Utilities/CubicHermiteInterpolatorInline.hh b/src/Utilities/CubicHermiteInterpolatorInline.hh index ece1a6fdb1..0c84c99cec 100644 --- a/src/Utilities/CubicHermiteInterpolatorInline.hh +++ b/src/Utilities/CubicHermiteInterpolatorInline.hh @@ -13,13 +13,8 @@ inline CubicHermiteInterpolator::CubicHermiteInterpolator(const double xmin, const double xmax, const size_t n, - const Func& F): - mN(n), - mXmin(xmin), - mXmax(xmax), - mXstep((xmax - xmin)/(n - 1u)), - mVals(2u*n) { - this->initialize(xmin, xmax, n, F); + const Func& F) { + initialize(xmin, xmax, n, F); } //------------------------------------------------------------------------------ @@ -31,13 +26,8 @@ CubicHermiteInterpolator::CubicHermiteInterpolator(const double xmin, const double xmax, const size_t n, const Func& F, - const GradFunc& Fgrad): - mN(n), - mXmin(xmin), - mXmax(xmax), - mXstep((xmax - xmin)/(n - 1u)), - mVals(2u*n) { - this->initialize(xmin, xmax, n, F, Fgrad); + const GradFunc& Fgrad) { + initialize(xmin, xmax, n, F, Fgrad); } //------------------------------------------------------------------------------ @@ -59,13 +49,13 @@ CubicHermiteInterpolator::initialize(const double xmin, mXmin = xmin; mXmax = xmax; mXstep = (xmax - xmin)/(n - 1u); - mVals.resize(2u*n); + mVals.allocate(2u*n, chai::CPU); // Compute the function values for (auto i = 0u; i < mN; ++i) mVals[i] = F(xmin + i*mXstep); // Initialize the gradient values - this->initializeGradientKnots(); + initializeGradientKnots(); // const auto dx = 0.001*mXstep; // for (auto i = 0u; i < mN; ++i) { @@ -96,7 +86,7 @@ CubicHermiteInterpolator::initialize(const double xmin, mXmin = xmin; mXmax = xmax; mXstep = (xmax - xmin)/(n - 1u); - mVals.resize(2u*n); + mVals.allocate(2u*n, chai::CPU); // Compute the function and gradient values for (auto i = 0u; i < mN; ++i) { @@ -109,9 +99,9 @@ CubicHermiteInterpolator::initialize(const double xmin, //------------------------------------------------------------------------------ // Interpolate for the given x value. //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::operator()(const double x) const { +CHIBase::operator()(const double x) const { if (x < mXmin) { return mVals[0] + mVals[mN]*(x - mXmin); } else if (x > mXmax) { @@ -122,9 +112,9 @@ CubicHermiteInterpolator::operator()(const double x) const { } } -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::operator()(const double x, +CHIBase::operator()(const double x, const size_t i0) const { REQUIRE(i0 <= mN - 2u); const auto t = std::max(0.0, std::min(1.0, (x - mXmin - i0*mXstep)/mXstep)); @@ -139,9 +129,9 @@ CubicHermiteInterpolator::operator()(const double x, //------------------------------------------------------------------------------ // Interpolate for dy/dx //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::prime(const double x) const { +CHIBase::prime(const double x) const { if (x < mXmin) { return mVals[mN]; } else if (x > mXmax) { @@ -152,9 +142,9 @@ CubicHermiteInterpolator::prime(const double x) const { } } -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::prime(const double x, +CHIBase::prime(const double x, const size_t i0) const { REQUIRE(i0 <= mN - 2u); const auto t = std::max(0.0, std::min(1.0, (x - mXmin - i0*mXstep)/mXstep)); @@ -167,9 +157,9 @@ CubicHermiteInterpolator::prime(const double x, //------------------------------------------------------------------------------ // Interpolate for d^2y/dx^2 //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::prime2(const double x) const { +CHIBase::prime2(const double x) const { if (x < mXmin or x > mXmax) { return 0.0; } else { @@ -178,9 +168,9 @@ CubicHermiteInterpolator::prime2(const double x) const { } } -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::prime2(const double x, +CHIBase::prime2(const double x, const size_t i0) const { REQUIRE(i0 <= mN - 2u); const auto t = std::max(0.0, std::min(1.0, (x - mXmin - i0*mXstep)/mXstep)); @@ -192,9 +182,9 @@ CubicHermiteInterpolator::prime2(const double x, //------------------------------------------------------------------------------ // Return the lower bound entry in the table for the given x coordinate //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline size_t -CubicHermiteInterpolator::lowerBound(const double x) const { +CHIBase::lowerBound(const double x) const { const auto result = std::min(mN - 2u, size_t(std::max(0.0, x - mXmin)/mXstep)); ENSURE(result <= mN - 2u); return result; @@ -203,61 +193,55 @@ CubicHermiteInterpolator::lowerBound(const double x) const { //------------------------------------------------------------------------------ // Hermite basis functions //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::h00(const double t) const { +CHIBase::h00(const double t) const { return (2.0*t - 3.0)*t*t + 1.0; } -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::h10(const double t) const { +CHIBase::h10(const double t) const { return (t - 2.0)*t*t + t; } -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::h01(const double t) const { +CHIBase::h01(const double t) const { return (3.0 - 2.0*t)*t*t; } -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::h11(const double t) const { +CHIBase::h11(const double t) const { return (t - 1.0)*t*t; } //------------------------------------------------------------------------------ // Data accessors //------------------------------------------------------------------------------ -inline +SPHERAL_HOST_DEVICE inline size_t -CubicHermiteInterpolator::size() const { +CHIBase::size() const { return mN; } -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::xmin() const { +CHIBase::xmin() const { return mXmin; } -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::xmax() const { +CHIBase::xmax() const { return mXmax; } -inline +SPHERAL_HOST_DEVICE inline double -CubicHermiteInterpolator::xstep() const { +CHIBase::xstep() const { return mXstep; } -inline -const std::vector& -CubicHermiteInterpolator::vals() const { - return mVals; -} - } diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index 078543acb1..7ff3b90ed7 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -17,15 +17,15 @@ namespace Spheral { -class QuadraticInterpolatorBase { +class QIBase { public: using ContainerType = typename chai::ManagedArray; //--------------------------- Public Interface ---------------------------// // Constructors, destructors - SPHERAL_HOST_DEVICE QuadraticInterpolatorBase() = default; + SPHERAL_HOST_DEVICE QIBase() = default; // Comparisons - SPHERAL_HOST_DEVICE bool operator==(const QuadraticInterpolatorBase& rhs) const; + SPHERAL_HOST_DEVICE bool operator==(const QIBase& rhs) const; // Interpolate for the y value SPHERAL_HOST_DEVICE double operator()(const double x) const; @@ -46,17 +46,17 @@ public: SPHERAL_HOST_DEVICE double xmax() const; // Maximum x coordinate for table SPHERAL_HOST_DEVICE double xstep() const; // delta x between tabulated values -protected: - SPHERAL_HOST QuadraticInterpolatorBase(size_t N1, - double xmin, - double xmax, - double xstep, - ContainerType const& vals) : + SPHERAL_HOST QIBase(size_t N1, + double xmin, + double xmax, + double xstep, + ContainerType const& vals) : mN1(N1), mXmin(xmin), mXmax(xmax), mXstep(xstep), mcoeffs(vals) { mcoeffs.registerTouch(chai::CPU); } +protected: //--------------------------- Private Interface --------------------------// // Member data size_t mN1 = 0u; @@ -66,7 +66,7 @@ protected: ContainerType mcoeffs; }; -class QuadraticInterpolator : public QuadraticInterpolatorBase { +class QuadraticInterpolator : public QIBase { public: using ContainerType = typename chai::ManagedArray; template @@ -79,20 +79,10 @@ public: void initialize(double xmin, double xmax, size_t n, const Func& f); void initialize(double xmin, double xmax, const std::vector& yvals); - template - QIView view(chai::ExecutionSpace space) { - return QIView(mN1, mXmin, mXmax, mXstep, mcoeffs); + QIBase view() { + return QIBase(mN1, mXmin, mXmax, mXstep, mcoeffs); } }; - -// For use on device -class QIView : public QuadraticInterpolatorBase { -public: - using ContainerType = typename chai::ManagedArray; - SPHERAL_HOST_DEVICE QIView() = default; - SPHERAL_HOST QIView(size_t N1, double xmin, double xmax, double xstep, ContainerType const& vals); - SPHERAL_HOST_DEVICE ~QIView() { } -}; } #include "QuadraticInterpolatorInline.hh" diff --git a/src/Utilities/QuadraticInterpolatorInline.hh b/src/Utilities/QuadraticInterpolatorInline.hh index 081a01e329..fb7e0a56e3 100644 --- a/src/Utilities/QuadraticInterpolatorInline.hh +++ b/src/Utilities/QuadraticInterpolatorInline.hh @@ -34,31 +34,19 @@ QuadraticInterpolator::initialize(double xmin, initialize(xmin, xmax, yvals); } -//------------------------------------------------------------------------------ -// Initialize QuadraticInterpolator -//------------------------------------------------------------------------------ -SPHERAL_HOST inline -QIView::QIView(size_t N1, - double xmin, - double xmax, - double xstep, - chai::ManagedArray const& vals) : - QuadraticInterpolatorBase(N1, xmin, xmax, xstep, vals) { -} - //------------------------------------------------------------------------------ // Interpolate for the given x value. //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -QuadraticInterpolatorBase::operator()(const double x) const { +QIBase::operator()(const double x) const { const auto i0 = lowerBound(x); return mcoeffs[i0] + (mcoeffs[i0 + 1] + mcoeffs[i0 + 2]*x)*x; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolatorBase::operator()(const double x, +QIBase::operator()(const double x, const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return mcoeffs[i0] + (mcoeffs[i0 + 1] + mcoeffs[i0 + 2]*x)*x; @@ -69,15 +57,15 @@ QuadraticInterpolatorBase::operator()(const double x, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -QuadraticInterpolatorBase::prime(const double x) const { +QIBase::prime(const double x) const { const auto i0 = lowerBound(x); return mcoeffs[i0 + 1] + 2.0*mcoeffs[i0 + 2]*x; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolatorBase::prime(const double x, - const size_t i0) const { +QIBase::prime(const double x, + const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return mcoeffs[i0 + 1] + 2.0*mcoeffs[i0 + 2]*x; } @@ -88,15 +76,15 @@ QuadraticInterpolatorBase::prime(const double x, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -QuadraticInterpolatorBase::prime2(const double x) const { +QIBase::prime2(const double x) const { const auto i0 = lowerBound(x); return 2.0*mcoeffs[i0 + 2]; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolatorBase::prime2(const double /*x*/, - const size_t i0) const { +QIBase::prime2(const double /*x*/, + const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return 2.0*mcoeffs[i0 + 2]; } @@ -106,7 +94,7 @@ QuadraticInterpolatorBase::prime2(const double /*x*/, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline size_t -QuadraticInterpolatorBase::lowerBound(const double x) const { +QIBase::lowerBound(const double x) const { const auto result = 3u*std::min(mN1, size_t(std::max(0.0, x - mXmin)/mXstep)); ENSURE(result <= 3u*mN1); return result; @@ -117,25 +105,25 @@ QuadraticInterpolatorBase::lowerBound(const double x) const { //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline size_t -QuadraticInterpolatorBase::size() const { +QIBase::size() const { return 3*(mN1 + 1u); } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolatorBase::xmin() const { +QIBase::xmin() const { return mXmin; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolatorBase::xmax() const { +QIBase::xmax() const { return mXmax; } SPHERAL_HOST_DEVICE inline double -QuadraticInterpolatorBase::xstep() const { +QIBase::xstep() const { return mXstep; } @@ -145,8 +133,8 @@ QuadraticInterpolatorBase::xstep() const { SPHERAL_HOST_DEVICE inline bool -QuadraticInterpolatorBase:: -operator==(const QuadraticInterpolatorBase& rhs) const { +QIBase:: +operator==(const QIBase& rhs) const { return ((mN1 == rhs.mN1) and (mXmin == rhs.mXmin) and (mXmax == rhs.mXmax) and diff --git a/tests/cpp/Utilities/CMakeLists.txt b/tests/cpp/Utilities/CMakeLists.txt index 119c30011c..a334eabe59 100644 --- a/tests/cpp/Utilities/CMakeLists.txt +++ b/tests/cpp/Utilities/CMakeLists.txt @@ -10,3 +10,10 @@ spheral_add_test( DEPENDS_ON Spheral_Utilities #DEBUG_LINKER ) + +spheral_add_test( + NAME chinterpolator_tests + SOURCES chinterpolator_tests.cc + DEPENDS_ON Spheral_Utilities + #DEBUG_LINKER +) diff --git a/tests/cpp/Utilities/chinterpolator_tests.cc b/tests/cpp/Utilities/chinterpolator_tests.cc new file mode 100644 index 0000000000..6be270efff --- /dev/null +++ b/tests/cpp/Utilities/chinterpolator_tests.cc @@ -0,0 +1,85 @@ +// Debug log printing can be quickly enabled for this unit test by uncommenting the +// definition below even if Spheral was not configured w/ SPHERAL_ENABLE_LOGGER=On. +// #define SPHERAL_ENABLE_LOGGER + +#include "chai/Types.hpp" +#include "test-basic-exec-policies.hh" +#include "test-utilities.hh" + +#include "Utilities/CubicHermiteInterpolator.hh" +#include +#include + +using QI = Spheral::CubicHermiteInterpolator; + +class CubicHermiteInterpolatorTest : public ::testing::Test { +public: + const double xmin = 10.; + const double xmax = 100.; + SPHERAL_HOST_DEVICE static double func(const double x) { + return 1. + x*(2. + 3.*x); + } + std::vector makeVec(size_t N) { + double xstep = (xmax - xmin)/((double)N - 1.); + std::vector yvals(N); + for (size_t i = 0; i < N; ++i) { + double x = xmin + xstep*(double)i; + yvals[i] = func(x); + } + return yvals; + } +}; + +// Setting up G Test for FieldList +TYPED_TEST_SUITE_P(CubicHermiteInterpolatorTypedTest); +template class CubicHermiteInterpolatorTypedTest : public CubicHermiteInterpolatorTest {}; + +// Test multiple FieldLists holding the same Field +GPU_TYPED_TEST_P(CubicHermiteInterpolatorTypedTest, FuncCtorTest) { + const size_t NV = 41; + const double xmin = gpu_this->xmin; + const double xmax = gpu_this->xmax; + QI qih(xmin, xmax, NV, gpu_this->func); + { + size_t N = qih.size(); + Spheral::CHIBase qi = qih.view(); + EXEC_IN_SPACE_BEGIN(TypeParam) + SPHERAL_ASSERT_EQ(qi.size(), N); + EXEC_IN_SPACE_END() + const double xstep = (xmax - xmin)/((double)NV - 1.); + RAJA::forall(TRS_UINT(0, NV), + [=] (size_t i) { + double x = xmin + xstep*(double)i; + double rval = gpu_this->func(x); + double ival = qi(x); + SPHERAL_ASSERT_FLOAT_EQ(rval, ival); + }); + } +} + +GPU_TYPED_TEST_P(CubicHermiteInterpolatorTypedTest, VecCtorTest) { + const size_t NV = 41; + std::vector yvals = gpu_this->makeVec(NV); + const double xmin = gpu_this->xmin; + const double xmax = gpu_this->xmax; + QI qih(xmin, xmax, yvals); + size_t N = qih.size(); + Spheral::CHIBase qi = qih.view(); + EXEC_IN_SPACE_BEGIN(TypeParam) + SPHERAL_ASSERT_EQ(qi.size(), N); + EXEC_IN_SPACE_END() + const double xstep = (xmax - xmin)/((double)NV - 1.); + RAJA::forall(TRS_UINT(0, NV), + [=] (size_t i) { + double x = xmin + xstep*(double)i; + double rval = gpu_this->func(x); + double ival = qi(x); + SPHERAL_ASSERT_FLOAT_EQ(rval, ival); + }); +} + +REGISTER_TYPED_TEST_SUITE_P(CubicHermiteInterpolatorTypedTest, FuncCtorTest, + VecCtorTest); + +INSTANTIATE_TYPED_TEST_SUITE_P(CubicHermiteInterpolator, CubicHermiteInterpolatorTypedTest, + typename Spheral::Test::Types, ); diff --git a/tests/cpp/Utilities/quadraticinterpolator_tests.cc b/tests/cpp/Utilities/quadraticinterpolator_tests.cc index 914c6fa023..0e75818ca5 100644 --- a/tests/cpp/Utilities/quadraticinterpolator_tests.cc +++ b/tests/cpp/Utilities/quadraticinterpolator_tests.cc @@ -42,14 +42,12 @@ GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, FuncCtorTest) { const double xmax = gpu_this->xmax; QI qih(xmin, xmax, NV, gpu_this->func); { - chai::ExecutionSpace space = chai::CPU; - if (typeid(TypeParam) != typeid(RAJA::seq_exec)) space = chai::GPU; size_t N = qih.size(); - Spheral::QIView qi = qih.view(space); + Spheral::QIBase qi = qih.view(); EXEC_IN_SPACE_BEGIN(TypeParam) SPHERAL_ASSERT_EQ(qi.size(), N); EXEC_IN_SPACE_END() - const double xstep = (xmax - xmin)/((double)NV - 1.); + const double xstep = (xmax - xmin)/((double)NV - 1.); RAJA::forall(TRS_UINT(0, NV), [=] (size_t i) { double x = xmin + xstep*(double)i; @@ -66,10 +64,8 @@ GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, VecCtorTest) { const double xmin = gpu_this->xmin; const double xmax = gpu_this->xmax; QI qih(xmin, xmax, yvals); - chai::ExecutionSpace space = chai::CPU; - if (typeid(TypeParam) != typeid(RAJA::seq_exec)) space = chai::GPU; size_t N = qih.size(); - Spheral::QIView qi = qih.view(space); + Spheral::QIBase qi = qih.view(); EXEC_IN_SPACE_BEGIN(TypeParam) SPHERAL_ASSERT_EQ(qi.size(), N); EXEC_IN_SPACE_END() From 8ac532caa110ca5a2a2f43cd21a1dbedeac77d74 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 29 Jul 2025 14:59:43 -0700 Subject: [PATCH 05/52] Added default constructors for QI and CHI, added the index operator for QI and CHI and changed the python test where necessary --- src/PYB11/Utilities/CubicHermiteInterpolator.py | 8 ++++++-- src/PYB11/Utilities/QuadraticInterpolator.py | 6 +++++- src/Utilities/CubicHermiteInterpolator.cc | 1 + src/Utilities/CubicHermiteInterpolator.hh | 4 +++- src/Utilities/CubicHermiteInterpolatorInline.hh | 8 ++++++++ src/Utilities/QuadraticInterpolator.hh | 3 +++ src/Utilities/QuadraticInterpolatorInline.hh | 10 +++++++++- tests/unit/Utilities/testCubicHermiteInterpolator.py | 4 ++-- 8 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/PYB11/Utilities/CubicHermiteInterpolator.py b/src/PYB11/Utilities/CubicHermiteInterpolator.py index ee5ef2fb8b..9020a34f13 100644 --- a/src/PYB11/Utilities/CubicHermiteInterpolator.py +++ b/src/PYB11/Utilities/CubicHermiteInterpolator.py @@ -8,7 +8,7 @@ class CubicHermiteInterpolator: "An (optionally monotonic) form of cubic Hermite interpolation." def pyinit(self): - "Default constuctor -- returns a non-functional interpolator until initialized" + "Default constructor -- returns a non-functional interpolator until initialized" return def pyinit_func(self, @@ -82,6 +82,10 @@ def __call__i0(self, "Returns the interpolated value (x)" return "double" + @PYB11cppname("operator[]") + def __getitem__(self, index="size_t"): + return "double" + @PYB11const def prime(self, x = "const double"): @@ -145,4 +149,4 @@ def h11(self, xmin = PYB11property(doc="Minimum x coordinate for table") xmax = PYB11property(doc="Maximum x coordinate for table") xstep = PYB11property(doc="delta x between tabulated values") - vals = PYB11property(doc="tabulated values and gradients (sequentially)") + diff --git a/src/PYB11/Utilities/QuadraticInterpolator.py b/src/PYB11/Utilities/QuadraticInterpolator.py index 6d1a4450d3..d0a0ffa824 100644 --- a/src/PYB11/Utilities/QuadraticInterpolator.py +++ b/src/PYB11/Utilities/QuadraticInterpolator.py @@ -9,7 +9,7 @@ class QuadraticInterpolator: Assumes the results is interpolated as y_interp = a + b*x + c*x^2""" def pyinit(self): - "Default constuctor -- returns a non-functional interpolator until initialized" + "Default constructor -- returns a non-functional interpolator until initialized" return def pyinit_func(self, @@ -49,6 +49,10 @@ def __call__(self, "Returns the interpolated value (x)" return "double" + @PYB11cppname("operator[]") + def __getitem__(self, index="size_t"): + return "double" + @PYB11const def prime(self, x = "const double"): diff --git a/src/Utilities/CubicHermiteInterpolator.cc b/src/Utilities/CubicHermiteInterpolator.cc index 47e55f72f6..2ffa0a572d 100644 --- a/src/Utilities/CubicHermiteInterpolator.cc +++ b/src/Utilities/CubicHermiteInterpolator.cc @@ -127,6 +127,7 @@ makeMonotonic() { } } } + mVals.registerTouch(chai::CPU); } //------------------------------------------------------------------------------ diff --git a/src/Utilities/CubicHermiteInterpolator.hh b/src/Utilities/CubicHermiteInterpolator.hh index 8e11164893..42c5fdba7c 100644 --- a/src/Utilities/CubicHermiteInterpolator.hh +++ b/src/Utilities/CubicHermiteInterpolator.hh @@ -28,6 +28,8 @@ public: SPHERAL_HOST_DEVICE double operator()(const double x) const; SPHERAL_HOST_DEVICE double prime(const double x) const; // First derivative SPHERAL_HOST_DEVICE double prime2(const double x) const; // Second derivative + // Index access + SPHERAL_HOST_DEVICE double operator[](const size_t i) const; // Same as above, but use a pre-computed table position (from lowerBound) SPHERAL_HOST_DEVICE double operator()(const double x, const size_t i0) const; @@ -87,7 +89,7 @@ public: CubicHermiteInterpolator(const double xmin, const double xmax, const std::vector& values); - CubicHermiteInterpolator(); + CubicHermiteInterpolator() = default; ~CubicHermiteInterpolator(); // (Re)initialize after construction, same options as construction diff --git a/src/Utilities/CubicHermiteInterpolatorInline.hh b/src/Utilities/CubicHermiteInterpolatorInline.hh index 0c84c99cec..2e952cf4f8 100644 --- a/src/Utilities/CubicHermiteInterpolatorInline.hh +++ b/src/Utilities/CubicHermiteInterpolatorInline.hh @@ -126,6 +126,14 @@ CHIBase::operator()(const double x, (t3 - t2)*mVals[mN + i0 + 1u])); // h11 } +SPHERAL_HOST_DEVICE inline +double +CHIBase::operator[](const size_t i) const { + REQUIRE(size() > 0); + REQUIRE(i < size()); + return mVals[i]; +} + //------------------------------------------------------------------------------ // Interpolate for dy/dx //------------------------------------------------------------------------------ diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index 7ff3b90ed7..cafaac6abf 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -31,6 +31,8 @@ public: SPHERAL_HOST_DEVICE double operator()(const double x) const; SPHERAL_HOST_DEVICE double prime(const double x) const; // First derivative SPHERAL_HOST_DEVICE double prime2(const double x) const; // Second derivative + // Index access + SPHERAL_HOST_DEVICE double operator[](const size_t i) const; // Same as above, but use a pre-computed table position (from lowerBound) SPHERAL_HOST_DEVICE double operator()(const double x, const size_t i0) const; @@ -72,6 +74,7 @@ public: template QuadraticInterpolator(double xmin, double xmax, size_t n, const Func& F); QuadraticInterpolator(double xmin, double xmax, const std::vector& yvals); + QuadraticInterpolator() = default; ~QuadraticInterpolator(); // Initialize after construction, either with a function or tabulated values diff --git a/src/Utilities/QuadraticInterpolatorInline.hh b/src/Utilities/QuadraticInterpolatorInline.hh index fb7e0a56e3..3ab8bedc3b 100644 --- a/src/Utilities/QuadraticInterpolatorInline.hh +++ b/src/Utilities/QuadraticInterpolatorInline.hh @@ -47,11 +47,19 @@ QIBase::operator()(const double x) const { SPHERAL_HOST_DEVICE inline double QIBase::operator()(const double x, - const size_t i0) const { + const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return mcoeffs[i0] + (mcoeffs[i0 + 1] + mcoeffs[i0 + 2]*x)*x; } +SPHERAL_HOST_DEVICE inline +double +QIBase::operator[](const size_t i) const { + REQUIRE(size() > 0); + REQUIRE(i < size()); + return mcoeffs[i]; +} + //------------------------------------------------------------------------------ // Interpolate the first derivative the given x value. //------------------------------------------------------------------------------ diff --git a/tests/unit/Utilities/testCubicHermiteInterpolator.py b/tests/unit/Utilities/testCubicHermiteInterpolator.py index 391a76319c..39481bdfdc 100644 --- a/tests/unit/Utilities/testCubicHermiteInterpolator.py +++ b/tests/unit/Utilities/testCubicHermiteInterpolator.py @@ -186,12 +186,12 @@ def tolFunc(params, x): # If requested, check for monotonicity in interpolation if checkMonotonicity: i0 = F.lowerBound(x) - passing = (F(x) - F.vals[i0])*(F(x) - F.vals[i0 + 1]) <= 0.0 + passing = (F(x) - F[i0])*(F(x) - F[i0 + 1]) <= 0.0 if not passing: #print(F.vals) self.plotem(x, xmin, xmax, func, F) self.assertTrue(passing, - "Failing monotonicity test for {}: F({}) = {} not in [{}, {}]".format(errorLabel, x, F(x), F.vals[i0], F.vals[i0 + 1])) + "Failing monotonicity test for {}: F({}) = {} not in [{}, {}]".format(errorLabel, x, F(x), F[i0], F[i0 + 1])) return From 9f7ef88871eed0c8608402840f09c95b040b686d Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 29 Jul 2025 15:29:54 -0700 Subject: [PATCH 06/52] Added move operation to QIBase and CHIBase --- src/Utilities/CubicHermiteInterpolator.hh | 1 + src/Utilities/QuadraticInterpolator.hh | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Utilities/CubicHermiteInterpolator.hh b/src/Utilities/CubicHermiteInterpolator.hh index 42c5fdba7c..726c167b0d 100644 --- a/src/Utilities/CubicHermiteInterpolator.hh +++ b/src/Utilities/CubicHermiteInterpolator.hh @@ -50,6 +50,7 @@ public: SPHERAL_HOST_DEVICE double xmin() const; // Minimum x coordinate for table SPHERAL_HOST_DEVICE double xmax() const; // Maximum x coordinate for table SPHERAL_HOST_DEVICE double xstep() const; // delta x between tabulated values + void move(chai::ExecutionSpace space) { mVals.move(space); } SPHERAL_HOST CHIBase(size_t N, double xmin, diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index cafaac6abf..291e8e4193 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -47,6 +47,7 @@ public: SPHERAL_HOST_DEVICE double xmin() const; // Minimum x coordinate for table SPHERAL_HOST_DEVICE double xmax() const; // Maximum x coordinate for table SPHERAL_HOST_DEVICE double xstep() const; // delta x between tabulated values + void move(chai::ExecutionSpace space) { mcoeffs.move(space); } SPHERAL_HOST QIBase(size_t N1, double xmin, From 440e72b23d375234c91b5953e4a3c4096c7d0272 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 31 Jul 2025 10:26:19 -0700 Subject: [PATCH 07/52] Added copy and assignment constructors and tests for the QI and CHI classes, made a std::vector in the QI and CHI value classes to hold the actual data instead of a MA --- src/Utilities/CubicHermiteInterpolator.cc | 48 +++++++++++---- src/Utilities/CubicHermiteInterpolator.hh | 14 ++--- .../CubicHermiteInterpolatorInline.hh | 42 ++++--------- src/Utilities/QuadraticInterpolator.cc | 39 ++++++++++-- src/Utilities/QuadraticInterpolator.hh | 6 ++ src/Utilities/QuadraticInterpolatorInline.hh | 3 +- tests/cpp/Utilities/chinterpolator_tests.cc | 60 ++++++++++++++----- .../Utilities/quadraticinterpolator_tests.cc | 37 ++++++++++-- 8 files changed, 172 insertions(+), 77 deletions(-) diff --git a/src/Utilities/CubicHermiteInterpolator.cc b/src/Utilities/CubicHermiteInterpolator.cc index 2ffa0a572d..998967cd91 100644 --- a/src/Utilities/CubicHermiteInterpolator.cc +++ b/src/Utilities/CubicHermiteInterpolator.cc @@ -16,6 +16,31 @@ namespace Spheral { +//------------------------------------------------------------------------------ +// Copy constructor +//------------------------------------------------------------------------------ +CubicHermiteInterpolator::CubicHermiteInterpolator(const CubicHermiteInterpolator& rhs) { + mN = rhs.mN; + mXmin = rhs.mXmin; + mXmax = rhs.mXmax; + mXstep = rhs.mXstep; + mVec = rhs.mVec; + initializeMA(); +} + +//------------------------------------------------------------------------------ +// Assignment constructor +//------------------------------------------------------------------------------ +CubicHermiteInterpolator& CubicHermiteInterpolator::operator=(const CubicHermiteInterpolator& rhs) { + mN = rhs.mN; + mXmin = rhs.mXmin; + mXmax = rhs.mXmax; + mXstep = rhs.mXstep; + mVec = rhs.mVec; + initializeMA(); + return *this; +} + //------------------------------------------------------------------------------ // Construct from explicit table of values //------------------------------------------------------------------------------ @@ -40,13 +65,12 @@ CubicHermiteInterpolator::initialize(const double xmin, mXmin = xmin; mXmax = xmax; mXstep = (xmax - xmin)/(mN - 1u); - mVals.allocate(2u*mN, chai::CPU); + mVec.resize(2u*mN); - for (size_t i = 0; i < mN; ++i) mVals[i] = yvals[i]; + for (size_t i = 0; i < mN; ++i) mVec[i] = yvals[i]; // Estimate the gradients at our lattice points initializeGradientKnots(); - // const auto dxInv = 1.0/mXstep; // for (auto i = 1u; i < mN - 1u; ++i) { // mVals[mN + i] = 0.5*(mVals[i + 1u] - mVals[i - 1u])*dxInv; @@ -62,12 +86,16 @@ CubicHermiteInterpolator::~CubicHermiteInterpolator() { mVals.free(); } +void +CubicHermiteInterpolator::initializeMA() { + mVals = chai::makeManagedArray(mVec.data(), mVec.size(), chai::CPU, false); +} + //------------------------------------------------------------------------------ // Equivalence //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE bool -CHIBase:: -operator==(const CHIBase& rhs) const { +CHIBase::operator==(const CHIBase& rhs) const { return ((mN == rhs.mN) and (mXmin == rhs.mXmin) and (mXmax == rhs.mXmax) and @@ -152,13 +180,13 @@ initializeGradientKnots() { A.insert(0, 1) = -1.0; A.insert(mN-1u, mN-2u) = -1.0; // Last row A.insert(mN-1u, mN-1u) = 4.0; - b(0) = 3.0*(mVals[1u] - mVals[0u])/mXstep; - b(mN-1u) = 3.0*(mVals[mN-1u] - mVals[mN-2u])/mXstep; + b(0) = 3.0*(mVec[1u] - mVec[0u])/mXstep; + b(mN-1u) = 3.0*(mVec[mN-1u] - mVec[mN-2u])/mXstep; for (auto k = 1u; k < mN-1u; ++k) { // rows A.insert(k, k-1u) = -0.5; A.insert(k, k) = 4.0; A.insert(k, k+1u) = -0.5; - b(k) = 1.5*(mVals[k+1u] - mVals[k-1u])/mXstep; + b(k) = 1.5*(mVec[k+1u] - mVec[k-1u])/mXstep; } // Solve for the gradient values @@ -167,8 +195,8 @@ initializeGradientKnots() { CHECK(solver.info() == Eigen::Success); const Eigen::VectorXd x = solver.solve(b); CHECK(solver.info() == Eigen::Success); - for (auto k = 0u; k < mN; ++k) mVals[mN + k] = x(k); - + for (auto k = 0u; k < mN; ++k) mVec[mN + k] = x(k); + initializeMA(); // Old crappy but simple method for comparison // mVals[mN] = (mVals[1] - mVals[0])/mXstep; // mVals[2*mN-1] = (mVals[mN-1] - mVals[mN-2])/mXstep; diff --git a/src/Utilities/CubicHermiteInterpolator.hh b/src/Utilities/CubicHermiteInterpolator.hh index 726c167b0d..f2b67b3e15 100644 --- a/src/Utilities/CubicHermiteInterpolator.hh +++ b/src/Utilities/CubicHermiteInterpolator.hh @@ -51,6 +51,7 @@ public: SPHERAL_HOST_DEVICE double xmax() const; // Maximum x coordinate for table SPHERAL_HOST_DEVICE double xstep() const; // delta x between tabulated values void move(chai::ExecutionSpace space) { mVals.move(space); } + SPHERAL_HOST_DEVICE double* data() const { return mVals.data(); } SPHERAL_HOST CHIBase(size_t N, double xmin, @@ -90,6 +91,8 @@ public: CubicHermiteInterpolator(const double xmin, const double xmax, const std::vector& values); + CubicHermiteInterpolator(const CubicHermiteInterpolator& rhs); + CubicHermiteInterpolator& operator=(const CubicHermiteInterpolator& rhs); CubicHermiteInterpolator() = default; ~CubicHermiteInterpolator(); @@ -120,17 +123,10 @@ private: //--------------------------- Private Interface --------------------------// // Initialize the gradient at the interpolation points based on the tabulated // interpolation values + std::vector mVec; void initializeGradientKnots(); + void initializeMA(); }; - -// For use on device -// class CHIView : public CHIBase { -// public: -// using ContainerType = typename chai::ManagedArray; -// SPHERAL_HOST_DEVICE CHIView() = default; -// SPHERAL_HOST CHIView(size_t N1, double xmin, double xmax, double xstep, ContainerType const& vals); -// SPHERAL_HOST_DEVICE ~CHIView() { } -// }; } #include "CubicHermiteInterpolatorInline.hh" diff --git a/src/Utilities/CubicHermiteInterpolatorInline.hh b/src/Utilities/CubicHermiteInterpolatorInline.hh index 2e952cf4f8..6f77612b94 100644 --- a/src/Utilities/CubicHermiteInterpolatorInline.hh +++ b/src/Utilities/CubicHermiteInterpolatorInline.hh @@ -40,31 +40,10 @@ CubicHermiteInterpolator::initialize(const double xmin, const double xmax, const size_t n, const Func& F) { - - // Preconditions - VERIFY2(n > 2u, "CubicHermiteInterpolator requires n >= 3 without a gradient function : n=" << n); - VERIFY2(xmax > xmin, "CubicHermiteInterpolator requires a positive domain: [" << xmin << " " << xmax << "]"); - - mN = n; - mXmin = xmin; - mXmax = xmax; - mXstep = (xmax - xmin)/(n - 1u); - mVals.allocate(2u*n, chai::CPU); - - // Compute the function values - for (auto i = 0u; i < mN; ++i) mVals[i] = F(xmin + i*mXstep); - - // Initialize the gradient values - initializeGradientKnots(); - - // const auto dx = 0.001*mXstep; - // for (auto i = 0u; i < mN; ++i) { - // const auto xi = xmin + i*mXstep; - // // mVals[mN + i] = (F(xi + dx) - F(xi - dx))/(2.0*dx); - // const auto x0 = std::max(xmin, xi - dx); - // const auto x1 = std::min(xmax, xi + dx); - // mVals[mN + i] = (F(x1) - F(x0))/(x1 - x0); - // } + double xstep = (xmax - xmin)/(n - 1u); + std::vector yvals(n); + for (auto i = 0u; i < n; ++i) yvals[i] = F(xmin + i*xstep); + initialize(xmin, xmax, yvals); } //------------------------------------------------------------------------------ @@ -86,14 +65,15 @@ CubicHermiteInterpolator::initialize(const double xmin, mXmin = xmin; mXmax = xmax; mXstep = (xmax - xmin)/(n - 1u); - mVals.allocate(2u*n, chai::CPU); + mVec.resize(2u*n); // Compute the function and gradient values for (auto i = 0u; i < mN; ++i) { const auto xi = xmin + i*mXstep; - mVals[i] = F(xi); - mVals[mN + i] = Fgrad(xi); + mVec[i] = F(xi); + mVec[mN + i] = Fgrad(xi); } + initializeMA(); } //------------------------------------------------------------------------------ @@ -115,7 +95,7 @@ CHIBase::operator()(const double x) const { SPHERAL_HOST_DEVICE inline double CHIBase::operator()(const double x, - const size_t i0) const { + const size_t i0) const { REQUIRE(i0 <= mN - 2u); const auto t = std::max(0.0, std::min(1.0, (x - mXmin - i0*mXstep)/mXstep)); const auto t2 = t*t; @@ -153,7 +133,7 @@ CHIBase::prime(const double x) const { SPHERAL_HOST_DEVICE inline double CHIBase::prime(const double x, - const size_t i0) const { + const size_t i0) const { REQUIRE(i0 <= mN - 2u); const auto t = std::max(0.0, std::min(1.0, (x - mXmin - i0*mXstep)/mXstep)); const auto t2 = t*t; @@ -179,7 +159,7 @@ CHIBase::prime2(const double x) const { SPHERAL_HOST_DEVICE inline double CHIBase::prime2(const double x, - const size_t i0) const { + const size_t i0) const { REQUIRE(i0 <= mN - 2u); const auto t = std::max(0.0, std::min(1.0, (x - mXmin - i0*mXstep)/mXstep)); return 2.0*(3.0*(2.0*t - 1.0)*(mVals[i0] - mVals[i0 + 1u])/mXstep + diff --git a/src/Utilities/QuadraticInterpolator.cc b/src/Utilities/QuadraticInterpolator.cc index 0e46aca6df..239c02ad46 100644 --- a/src/Utilities/QuadraticInterpolator.cc +++ b/src/Utilities/QuadraticInterpolator.cc @@ -13,6 +13,31 @@ namespace Spheral { +//------------------------------------------------------------------------------ +// Copy constructor +//------------------------------------------------------------------------------ +QuadraticInterpolator::QuadraticInterpolator(const QuadraticInterpolator& rhs) { + mN1 = rhs.mN1; + mXmin = rhs.mXmin; + mXmax = rhs.mXmax; + mXstep = rhs.mXstep; + mVec = rhs.mVec; + initializeMA(); +} + +//------------------------------------------------------------------------------ +// Assignment constructor +//------------------------------------------------------------------------------ +QuadraticInterpolator& QuadraticInterpolator::operator=(const QuadraticInterpolator& rhs) { + mN1 = rhs.mN1; + mXmin = rhs.mXmin; + mXmax = rhs.mXmax; + mXstep = rhs.mXstep; + mVec = rhs.mVec; + initializeMA(); + return *this; +} + //------------------------------------------------------------------------------ // Constructor with sampled values //------------------------------------------------------------------------------ @@ -37,7 +62,7 @@ QuadraticInterpolator::initialize(double xmin, size_t N1 = (n - 1u)/2u - 1u; // Maximum index into arrays double xstep = (xmax - xmin)/(N1 + 1u); size_t N = 3*(N1 + 1u); - mcoeffs.allocate(N, chai::CPU); + mVec.resize(N); typedef Eigen::Matrix EMatrix; typedef Eigen::Matrix EVector; @@ -57,14 +82,20 @@ QuadraticInterpolator::initialize(double xmin, 1.0, x2, x2*x2; B << yvals[2u*i0], yvals[2u*i0 + 1u], yvals[2u*i0 + 2u]; X = A.inverse()*B; - mcoeffs[3*i0 ] = X(0); - mcoeffs[3*i0 + 1u] = X(1); - mcoeffs[3*i0 + 2u] = X(2); + mVec[3*i0 ] = X(0); + mVec[3*i0 + 1u] = X(1); + mVec[3*i0 + 2u] = X(2); } mN1 = N1; mXmin = xmin; mXmax = xmax; mXstep = xstep; + initializeMA(); +} + +void +QuadraticInterpolator::initializeMA() { + mcoeffs = chai::makeManagedArray(mVec.data(), mVec.size(), chai::CPU, false); } //------------------------------------------------------------------------------ diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index 291e8e4193..989c3a06f9 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -48,6 +48,7 @@ public: SPHERAL_HOST_DEVICE double xmax() const; // Maximum x coordinate for table SPHERAL_HOST_DEVICE double xstep() const; // delta x between tabulated values void move(chai::ExecutionSpace space) { mcoeffs.move(space); } + SPHERAL_HOST_DEVICE double* data() const { return mcoeffs.data(); } SPHERAL_HOST QIBase(size_t N1, double xmin, @@ -77,6 +78,8 @@ public: QuadraticInterpolator(double xmin, double xmax, const std::vector& yvals); QuadraticInterpolator() = default; ~QuadraticInterpolator(); + QuadraticInterpolator(const QuadraticInterpolator& rhs); + QuadraticInterpolator& operator=(const QuadraticInterpolator& rhs); // Initialize after construction, either with a function or tabulated values template @@ -86,6 +89,9 @@ public: QIBase view() { return QIBase(mN1, mXmin, mXmax, mXstep, mcoeffs); } +private: + std::vector mVec; + void initializeMA(); }; } diff --git a/src/Utilities/QuadraticInterpolatorInline.hh b/src/Utilities/QuadraticInterpolatorInline.hh index 3ab8bedc3b..a8c8df56d9 100644 --- a/src/Utilities/QuadraticInterpolatorInline.hh +++ b/src/Utilities/QuadraticInterpolatorInline.hh @@ -141,8 +141,7 @@ QIBase::xstep() const { SPHERAL_HOST_DEVICE inline bool -QIBase:: -operator==(const QIBase& rhs) const { +QIBase::operator==(const QIBase& rhs) const { return ((mN1 == rhs.mN1) and (mXmin == rhs.mXmin) and (mXmax == rhs.mXmax) and diff --git a/tests/cpp/Utilities/chinterpolator_tests.cc b/tests/cpp/Utilities/chinterpolator_tests.cc index 6be270efff..f15e0597e9 100644 --- a/tests/cpp/Utilities/chinterpolator_tests.cc +++ b/tests/cpp/Utilities/chinterpolator_tests.cc @@ -2,6 +2,7 @@ // definition below even if Spheral was not configured w/ SPHERAL_ENABLE_LOGGER=On. // #define SPHERAL_ENABLE_LOGGER +#include "chai/ExecutionSpaces.hpp" #include "chai/Types.hpp" #include "test-basic-exec-policies.hh" #include "test-utilities.hh" @@ -10,10 +11,11 @@ #include #include -using QI = Spheral::CubicHermiteInterpolator; +using CHI = Spheral::CubicHermiteInterpolator; class CubicHermiteInterpolatorTest : public ::testing::Test { public: + const size_t NV = 41; const double xmin = 10.; const double xmax = 100.; SPHERAL_HOST_DEVICE static double func(const double x) { @@ -30,55 +32,81 @@ class CubicHermiteInterpolatorTest : public ::testing::Test { } }; -// Setting up G Test for FieldList +// Setting up G Test for CHI TYPED_TEST_SUITE_P(CubicHermiteInterpolatorTypedTest); template class CubicHermiteInterpolatorTypedTest : public CubicHermiteInterpolatorTest {}; -// Test multiple FieldLists holding the same Field +// Test copy and assignment constructors +GPU_TYPED_TEST_P(CubicHermiteInterpolatorTypedTest, CopyAssign) { + const size_t NV = gpu_this->NV; + CHI chiref(gpu_this->xmin, gpu_this->xmax, NV, gpu_this->func); + { + CHI chi1(chiref); + SPHERAL_ASSERT_EQ(chi1.size(), chiref.size()); + CHI chi2 = chiref; + SPHERAL_ASSERT_EQ(chi2.size(), chiref.size()); + Spheral::CHIBase chi1_view = chi1.view(); + Spheral::CHIBase chi2_view = chi2.view(); + Spheral::CHIBase chiref_view = chiref; + // Ensure the underlying data pointer is different from the initial CHI + EXEC_IN_SPACE_BEGIN(TypeParam) + SPHERAL_ASSERT_NE(chi1_view.data(), chiref_view.data()); + SPHERAL_ASSERT_NE(chi2_view.data(), chiref_view.data()); + EXEC_IN_SPACE_END() + RAJA::forall(TRS_UINT(0, NV), + [=] (size_t i) { + SPHERAL_ASSERT_EQ(chi1_view[i], chiref_view[i]); + SPHERAL_ASSERT_EQ(chi2_view[i], chiref_view[i]); + }); + } +} + +// Test initialize using a func GPU_TYPED_TEST_P(CubicHermiteInterpolatorTypedTest, FuncCtorTest) { - const size_t NV = 41; + const size_t NV = gpu_this->NV; const double xmin = gpu_this->xmin; const double xmax = gpu_this->xmax; - QI qih(xmin, xmax, NV, gpu_this->func); + CHI chih(xmin, xmax, NV, gpu_this->func); { - size_t N = qih.size(); - Spheral::CHIBase qi = qih.view(); + size_t N = chih.size(); + Spheral::CHIBase chi = chih.view(); EXEC_IN_SPACE_BEGIN(TypeParam) - SPHERAL_ASSERT_EQ(qi.size(), N); + SPHERAL_ASSERT_EQ(chi.size(), N); EXEC_IN_SPACE_END() const double xstep = (xmax - xmin)/((double)NV - 1.); RAJA::forall(TRS_UINT(0, NV), [=] (size_t i) { double x = xmin + xstep*(double)i; double rval = gpu_this->func(x); - double ival = qi(x); + double ival = chi(x); SPHERAL_ASSERT_FLOAT_EQ(rval, ival); }); } } +// Test initialize using a vector GPU_TYPED_TEST_P(CubicHermiteInterpolatorTypedTest, VecCtorTest) { - const size_t NV = 41; + const size_t NV = gpu_this->NV; std::vector yvals = gpu_this->makeVec(NV); const double xmin = gpu_this->xmin; const double xmax = gpu_this->xmax; - QI qih(xmin, xmax, yvals); - size_t N = qih.size(); - Spheral::CHIBase qi = qih.view(); + CHI chih(xmin, xmax, yvals); + size_t N = chih.size(); + Spheral::CHIBase chi = chih.view(); EXEC_IN_SPACE_BEGIN(TypeParam) - SPHERAL_ASSERT_EQ(qi.size(), N); + SPHERAL_ASSERT_EQ(chi.size(), N); EXEC_IN_SPACE_END() const double xstep = (xmax - xmin)/((double)NV - 1.); RAJA::forall(TRS_UINT(0, NV), [=] (size_t i) { double x = xmin + xstep*(double)i; double rval = gpu_this->func(x); - double ival = qi(x); + double ival = chi(x); SPHERAL_ASSERT_FLOAT_EQ(rval, ival); }); } -REGISTER_TYPED_TEST_SUITE_P(CubicHermiteInterpolatorTypedTest, FuncCtorTest, +REGISTER_TYPED_TEST_SUITE_P(CubicHermiteInterpolatorTypedTest, CopyAssign, FuncCtorTest, VecCtorTest); INSTANTIATE_TYPED_TEST_SUITE_P(CubicHermiteInterpolator, CubicHermiteInterpolatorTypedTest, diff --git a/tests/cpp/Utilities/quadraticinterpolator_tests.cc b/tests/cpp/Utilities/quadraticinterpolator_tests.cc index 0e75818ca5..0fd746be6d 100644 --- a/tests/cpp/Utilities/quadraticinterpolator_tests.cc +++ b/tests/cpp/Utilities/quadraticinterpolator_tests.cc @@ -15,6 +15,7 @@ using QI = Spheral::QuadraticInterpolator; class QuadraticInterpolatorTest : public ::testing::Test { public: + const size_t NV = 41; const double xmin = 10.; const double xmax = 100.; SPHERAL_HOST_DEVICE static double func(const double x) { @@ -31,13 +32,38 @@ class QuadraticInterpolatorTest : public ::testing::Test { } }; -// Setting up G Test for FieldList +// Setting up G Test for QI TYPED_TEST_SUITE_P(QuadraticInterpolatorTypedTest); template class QuadraticInterpolatorTypedTest : public QuadraticInterpolatorTest {}; -// Test multiple FieldLists holding the same Field +// Test copy and assignment constructors +GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, CopyAssign) { + const size_t NV = gpu_this->NV; + QI qiref(gpu_this->xmin, gpu_this->xmax, NV, gpu_this->func); + { + QI qi1(qiref); + SPHERAL_ASSERT_EQ(qi1.size(), qiref.size()); + QI qi2 = qiref; + SPHERAL_ASSERT_EQ(qi2.size(), qiref.size()); + Spheral::QIBase qi1_view = qi1.view(); + Spheral::QIBase qi2_view = qi2.view(); + Spheral::QIBase qiref_view = qiref; + // Ensure the underlying data pointer is different from the initial QI + EXEC_IN_SPACE_BEGIN(TypeParam) + SPHERAL_ASSERT_NE(qi1_view.data(), qiref_view.data()); + SPHERAL_ASSERT_NE(qi2_view.data(), qiref_view.data()); + EXEC_IN_SPACE_END() + RAJA::forall(TRS_UINT(0, NV), + [=] (size_t i) { + SPHERAL_ASSERT_EQ(qi1_view[i], qiref_view[i]); + SPHERAL_ASSERT_EQ(qi2_view[i], qiref_view[i]); + }); + } +} + +// Test initialize using a func GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, FuncCtorTest) { - const size_t NV = 41; + const size_t NV = gpu_this->NV; const double xmin = gpu_this->xmin; const double xmax = gpu_this->xmax; QI qih(xmin, xmax, NV, gpu_this->func); @@ -58,8 +84,9 @@ GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, FuncCtorTest) { } } +// Test initialize using a vector GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, VecCtorTest) { - const size_t NV = 41; + const size_t NV = gpu_this->NV; std::vector yvals = gpu_this->makeVec(NV); const double xmin = gpu_this->xmin; const double xmax = gpu_this->xmax; @@ -79,7 +106,7 @@ GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, VecCtorTest) { }); } -REGISTER_TYPED_TEST_SUITE_P(QuadraticInterpolatorTypedTest, FuncCtorTest, +REGISTER_TYPED_TEST_SUITE_P(QuadraticInterpolatorTypedTest, CopyAssign, FuncCtorTest, VecCtorTest); INSTANTIATE_TYPED_TEST_SUITE_P(QuadraticInterpolator, QuadraticInterpolatorTypedTest, From 653bf652b84a79487750581a27ca51a5e9085d22 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 31 Jul 2025 11:59:42 -0700 Subject: [PATCH 08/52] Add explicit calls to base copy and assignment constructors in QI and CHI classes --- src/Utilities/CubicHermiteInterpolator.cc | 21 +++++++++------------ src/Utilities/QuadraticInterpolator.cc | 22 ++++++++++------------ 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/Utilities/CubicHermiteInterpolator.cc b/src/Utilities/CubicHermiteInterpolator.cc index 998967cd91..298dd31552 100644 --- a/src/Utilities/CubicHermiteInterpolator.cc +++ b/src/Utilities/CubicHermiteInterpolator.cc @@ -19,11 +19,8 @@ namespace Spheral { //------------------------------------------------------------------------------ // Copy constructor //------------------------------------------------------------------------------ -CubicHermiteInterpolator::CubicHermiteInterpolator(const CubicHermiteInterpolator& rhs) { - mN = rhs.mN; - mXmin = rhs.mXmin; - mXmax = rhs.mXmax; - mXstep = rhs.mXstep; +CubicHermiteInterpolator::CubicHermiteInterpolator(const CubicHermiteInterpolator& rhs) : + CHIBase(rhs) { mVec = rhs.mVec; initializeMA(); } @@ -31,13 +28,13 @@ CubicHermiteInterpolator::CubicHermiteInterpolator(const CubicHermiteInterpolato //------------------------------------------------------------------------------ // Assignment constructor //------------------------------------------------------------------------------ -CubicHermiteInterpolator& CubicHermiteInterpolator::operator=(const CubicHermiteInterpolator& rhs) { - mN = rhs.mN; - mXmin = rhs.mXmin; - mXmax = rhs.mXmax; - mXstep = rhs.mXstep; - mVec = rhs.mVec; - initializeMA(); +CubicHermiteInterpolator& +CubicHermiteInterpolator::operator=(const CubicHermiteInterpolator& rhs) { + if (this != &rhs) { + CHIBase::operator=(rhs); + mVec = rhs.mVec; + initializeMA(); + } return *this; } diff --git a/src/Utilities/QuadraticInterpolator.cc b/src/Utilities/QuadraticInterpolator.cc index 239c02ad46..3ebe6563bc 100644 --- a/src/Utilities/QuadraticInterpolator.cc +++ b/src/Utilities/QuadraticInterpolator.cc @@ -16,11 +16,9 @@ namespace Spheral { //------------------------------------------------------------------------------ // Copy constructor //------------------------------------------------------------------------------ -QuadraticInterpolator::QuadraticInterpolator(const QuadraticInterpolator& rhs) { - mN1 = rhs.mN1; - mXmin = rhs.mXmin; - mXmax = rhs.mXmax; - mXstep = rhs.mXstep; +QuadraticInterpolator::QuadraticInterpolator(const QuadraticInterpolator& rhs) + : + QIBase(rhs) { mVec = rhs.mVec; initializeMA(); } @@ -28,13 +26,13 @@ QuadraticInterpolator::QuadraticInterpolator(const QuadraticInterpolator& rhs) { //------------------------------------------------------------------------------ // Assignment constructor //------------------------------------------------------------------------------ -QuadraticInterpolator& QuadraticInterpolator::operator=(const QuadraticInterpolator& rhs) { - mN1 = rhs.mN1; - mXmin = rhs.mXmin; - mXmax = rhs.mXmax; - mXstep = rhs.mXstep; - mVec = rhs.mVec; - initializeMA(); +QuadraticInterpolator& +QuadraticInterpolator::operator=(const QuadraticInterpolator& rhs) { + if (this != &rhs) { + QIBase::operator=(rhs); + mVec = rhs.mVec; + initializeMA(); + } return *this; } From 891ef903609d5c68655bb0a2a98850fead15bc61 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 1 Aug 2025 10:45:28 -0700 Subject: [PATCH 09/52] Updated RELEASE_NOTES to test new mirror --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 316570b819..5bc8823c10 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,7 +5,7 @@ Version vYYYY.MM.p -- Release date YYYY-MM-DD Notable changes include: * New features / API changes: - * GPU Poring Effort: + * GPU Porting Effort: * Spheral::FieldView allows for implicit data migration of Spheral::Field data. * Implements FieldView datatypes as handles to be used for migrating data to and from the GPU. * Unit testing for semantic behaviour, H/D copy, and allocation / deallocaiton across a range of common pattens. From b427ef6d3e748390a11c819268b9ac949453241c Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 14 Aug 2025 16:54:30 -0700 Subject: [PATCH 10/52] Changed QIBase and CHIBase to QIView and CHIView --- src/Utilities/CubicHermiteInterpolator.cc | 19 +++++------ src/Utilities/CubicHermiteInterpolator.hh | 14 ++++---- .../CubicHermiteInterpolatorInline.hh | 32 +++++++++---------- src/Utilities/QuadraticInterpolator.cc | 4 +-- src/Utilities/QuadraticInterpolator.hh | 14 ++++---- src/Utilities/QuadraticInterpolatorInline.hh | 26 +++++++-------- tests/cpp/Utilities/chinterpolator_tests.cc | 10 +++--- .../Utilities/quadraticinterpolator_tests.cc | 10 +++--- 8 files changed, 63 insertions(+), 66 deletions(-) diff --git a/src/Utilities/CubicHermiteInterpolator.cc b/src/Utilities/CubicHermiteInterpolator.cc index 298dd31552..72155034bc 100644 --- a/src/Utilities/CubicHermiteInterpolator.cc +++ b/src/Utilities/CubicHermiteInterpolator.cc @@ -20,7 +20,7 @@ namespace Spheral { // Copy constructor //------------------------------------------------------------------------------ CubicHermiteInterpolator::CubicHermiteInterpolator(const CubicHermiteInterpolator& rhs) : - CHIBase(rhs) { + CHIView(rhs) { mVec = rhs.mVec; initializeMA(); } @@ -31,7 +31,7 @@ CubicHermiteInterpolator::CubicHermiteInterpolator(const CubicHermiteInterpolato CubicHermiteInterpolator& CubicHermiteInterpolator::operator=(const CubicHermiteInterpolator& rhs) { if (this != &rhs) { - CHIBase::operator=(rhs); + CHIView::operator=(rhs); mVec = rhs.mVec; initializeMA(); } @@ -41,10 +41,9 @@ CubicHermiteInterpolator::operator=(const CubicHermiteInterpolator& rhs) { //------------------------------------------------------------------------------ // Construct from explicit table of values //------------------------------------------------------------------------------ -CubicHermiteInterpolator:: -CubicHermiteInterpolator(const double xmin, - const double xmax, - const std::vector& yvals) { +CubicHermiteInterpolator::CubicHermiteInterpolator(const double xmin, + const double xmax, + const std::vector& yvals) { initialize(xmin, xmax, yvals); } @@ -92,7 +91,7 @@ CubicHermiteInterpolator::initializeMA() { // Equivalence //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE bool -CHIBase::operator==(const CHIBase& rhs) const { +CHIView::operator==(const CHIView& rhs) const { return ((mN == rhs.mN) and (mXmin == rhs.mXmin) and (mXmax == rhs.mXmax) and @@ -106,8 +105,7 @@ CHIBase::operator==(const CHIBase& rhs) const { // https://en.wikipedia.org/wiki/Monotone_cubic_interpolation //------------------------------------------------------------------------------ void -CubicHermiteInterpolator:: -makeMonotonic() { +CubicHermiteInterpolator::makeMonotonic() { // Compute the slope between tabulated values. std::vector cgrad(mN - 1u); @@ -164,8 +162,7 @@ makeMonotonic() { // Note: the function values must have already been set in mVals[0,n-1]! //------------------------------------------------------------------------------ void -CubicHermiteInterpolator:: -initializeGradientKnots() { +CubicHermiteInterpolator::initializeGradientKnots() { // Set up to solve using Eigen's linear solvers (Ax=b) // We know the matrix A is tridiagonal: sparse with the only non-zero elements diff --git a/src/Utilities/CubicHermiteInterpolator.hh b/src/Utilities/CubicHermiteInterpolator.hh index f2b67b3e15..dc5dd3b5fe 100644 --- a/src/Utilities/CubicHermiteInterpolator.hh +++ b/src/Utilities/CubicHermiteInterpolator.hh @@ -15,14 +15,14 @@ #include namespace Spheral { -class CHIBase { +class CHIView { public: using ContainerType = typename chai::ManagedArray; //--------------------------- Public Interface ---------------------------// // Constructors, destructors - SPHERAL_HOST_DEVICE CHIBase() = default; + SPHERAL_HOST_DEVICE CHIView() = default; // Comparisons - SPHERAL_HOST_DEVICE bool operator==(const CHIBase& rhs) const; + SPHERAL_HOST_DEVICE bool operator==(const CHIView& rhs) const; // Interpolate for the y value SPHERAL_HOST_DEVICE double operator()(const double x) const; @@ -53,7 +53,7 @@ public: void move(chai::ExecutionSpace space) { mVals.move(space); } SPHERAL_HOST_DEVICE double* data() const { return mVals.data(); } - SPHERAL_HOST CHIBase(size_t N, + SPHERAL_HOST CHIView(size_t N, double xmin, double xmax, double xstep, @@ -73,7 +73,7 @@ protected: ContainerType mVals; }; -class CubicHermiteInterpolator : public CHIBase { +class CubicHermiteInterpolator : public CHIView { public: //--------------------------- Public Interface ---------------------------// // Constructors, destructors @@ -115,8 +115,8 @@ public: // Force interpolation to be monotonic (may introduce structure between tabulated points) void makeMonotonic(); - CHIBase view() { - return CHIBase(mN, mXmin, mXmax, mXstep, mVals); + CHIView view() { + return CHIView(mN, mXmin, mXmax, mXstep, mVals); } private: diff --git a/src/Utilities/CubicHermiteInterpolatorInline.hh b/src/Utilities/CubicHermiteInterpolatorInline.hh index 6f77612b94..946b31c7e1 100644 --- a/src/Utilities/CubicHermiteInterpolatorInline.hh +++ b/src/Utilities/CubicHermiteInterpolatorInline.hh @@ -81,7 +81,7 @@ CubicHermiteInterpolator::initialize(const double xmin, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -CHIBase::operator()(const double x) const { +CHIView::operator()(const double x) const { if (x < mXmin) { return mVals[0] + mVals[mN]*(x - mXmin); } else if (x > mXmax) { @@ -94,7 +94,7 @@ CHIBase::operator()(const double x) const { SPHERAL_HOST_DEVICE inline double -CHIBase::operator()(const double x, +CHIView::operator()(const double x, const size_t i0) const { REQUIRE(i0 <= mN - 2u); const auto t = std::max(0.0, std::min(1.0, (x - mXmin - i0*mXstep)/mXstep)); @@ -108,7 +108,7 @@ CHIBase::operator()(const double x, SPHERAL_HOST_DEVICE inline double -CHIBase::operator[](const size_t i) const { +CHIView::operator[](const size_t i) const { REQUIRE(size() > 0); REQUIRE(i < size()); return mVals[i]; @@ -119,7 +119,7 @@ CHIBase::operator[](const size_t i) const { //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -CHIBase::prime(const double x) const { +CHIView::prime(const double x) const { if (x < mXmin) { return mVals[mN]; } else if (x > mXmax) { @@ -132,7 +132,7 @@ CHIBase::prime(const double x) const { SPHERAL_HOST_DEVICE inline double -CHIBase::prime(const double x, +CHIView::prime(const double x, const size_t i0) const { REQUIRE(i0 <= mN - 2u); const auto t = std::max(0.0, std::min(1.0, (x - mXmin - i0*mXstep)/mXstep)); @@ -147,7 +147,7 @@ CHIBase::prime(const double x, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -CHIBase::prime2(const double x) const { +CHIView::prime2(const double x) const { if (x < mXmin or x > mXmax) { return 0.0; } else { @@ -158,7 +158,7 @@ CHIBase::prime2(const double x) const { SPHERAL_HOST_DEVICE inline double -CHIBase::prime2(const double x, +CHIView::prime2(const double x, const size_t i0) const { REQUIRE(i0 <= mN - 2u); const auto t = std::max(0.0, std::min(1.0, (x - mXmin - i0*mXstep)/mXstep)); @@ -172,7 +172,7 @@ CHIBase::prime2(const double x, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline size_t -CHIBase::lowerBound(const double x) const { +CHIView::lowerBound(const double x) const { const auto result = std::min(mN - 2u, size_t(std::max(0.0, x - mXmin)/mXstep)); ENSURE(result <= mN - 2u); return result; @@ -183,25 +183,25 @@ CHIBase::lowerBound(const double x) const { //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -CHIBase::h00(const double t) const { +CHIView::h00(const double t) const { return (2.0*t - 3.0)*t*t + 1.0; } SPHERAL_HOST_DEVICE inline double -CHIBase::h10(const double t) const { +CHIView::h10(const double t) const { return (t - 2.0)*t*t + t; } SPHERAL_HOST_DEVICE inline double -CHIBase::h01(const double t) const { +CHIView::h01(const double t) const { return (3.0 - 2.0*t)*t*t; } SPHERAL_HOST_DEVICE inline double -CHIBase::h11(const double t) const { +CHIView::h11(const double t) const { return (t - 1.0)*t*t; } @@ -210,25 +210,25 @@ CHIBase::h11(const double t) const { //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline size_t -CHIBase::size() const { +CHIView::size() const { return mN; } SPHERAL_HOST_DEVICE inline double -CHIBase::xmin() const { +CHIView::xmin() const { return mXmin; } SPHERAL_HOST_DEVICE inline double -CHIBase::xmax() const { +CHIView::xmax() const { return mXmax; } SPHERAL_HOST_DEVICE inline double -CHIBase::xstep() const { +CHIView::xstep() const { return mXstep; } diff --git a/src/Utilities/QuadraticInterpolator.cc b/src/Utilities/QuadraticInterpolator.cc index 3ebe6563bc..104689716c 100644 --- a/src/Utilities/QuadraticInterpolator.cc +++ b/src/Utilities/QuadraticInterpolator.cc @@ -18,7 +18,7 @@ namespace Spheral { //------------------------------------------------------------------------------ QuadraticInterpolator::QuadraticInterpolator(const QuadraticInterpolator& rhs) : - QIBase(rhs) { + QIView(rhs) { mVec = rhs.mVec; initializeMA(); } @@ -29,7 +29,7 @@ QuadraticInterpolator::QuadraticInterpolator(const QuadraticInterpolator& rhs) QuadraticInterpolator& QuadraticInterpolator::operator=(const QuadraticInterpolator& rhs) { if (this != &rhs) { - QIBase::operator=(rhs); + QIView::operator=(rhs); mVec = rhs.mVec; initializeMA(); } diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index 989c3a06f9..e06abf0ab3 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -17,15 +17,15 @@ namespace Spheral { -class QIBase { +class QIView { public: using ContainerType = typename chai::ManagedArray; //--------------------------- Public Interface ---------------------------// // Constructors, destructors - SPHERAL_HOST_DEVICE QIBase() = default; + SPHERAL_HOST_DEVICE QIView() = default; // Comparisons - SPHERAL_HOST_DEVICE bool operator==(const QIBase& rhs) const; + SPHERAL_HOST_DEVICE bool operator==(const QIView& rhs) const; // Interpolate for the y value SPHERAL_HOST_DEVICE double operator()(const double x) const; @@ -50,7 +50,7 @@ public: void move(chai::ExecutionSpace space) { mcoeffs.move(space); } SPHERAL_HOST_DEVICE double* data() const { return mcoeffs.data(); } - SPHERAL_HOST QIBase(size_t N1, + SPHERAL_HOST QIView(size_t N1, double xmin, double xmax, double xstep, @@ -70,7 +70,7 @@ protected: ContainerType mcoeffs; }; -class QuadraticInterpolator : public QIBase { +class QuadraticInterpolator : public QIView { public: using ContainerType = typename chai::ManagedArray; template @@ -86,8 +86,8 @@ public: void initialize(double xmin, double xmax, size_t n, const Func& f); void initialize(double xmin, double xmax, const std::vector& yvals); - QIBase view() { - return QIBase(mN1, mXmin, mXmax, mXstep, mcoeffs); + QIView view() { + return QIView(mN1, mXmin, mXmax, mXstep, mcoeffs); } private: std::vector mVec; diff --git a/src/Utilities/QuadraticInterpolatorInline.hh b/src/Utilities/QuadraticInterpolatorInline.hh index a8c8df56d9..cc3cb5cb07 100644 --- a/src/Utilities/QuadraticInterpolatorInline.hh +++ b/src/Utilities/QuadraticInterpolatorInline.hh @@ -39,14 +39,14 @@ QuadraticInterpolator::initialize(double xmin, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -QIBase::operator()(const double x) const { +QIView::operator()(const double x) const { const auto i0 = lowerBound(x); return mcoeffs[i0] + (mcoeffs[i0 + 1] + mcoeffs[i0 + 2]*x)*x; } SPHERAL_HOST_DEVICE inline double -QIBase::operator()(const double x, +QIView::operator()(const double x, const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return mcoeffs[i0] + (mcoeffs[i0 + 1] + mcoeffs[i0 + 2]*x)*x; @@ -54,7 +54,7 @@ QIBase::operator()(const double x, SPHERAL_HOST_DEVICE inline double -QIBase::operator[](const size_t i) const { +QIView::operator[](const size_t i) const { REQUIRE(size() > 0); REQUIRE(i < size()); return mcoeffs[i]; @@ -65,14 +65,14 @@ QIBase::operator[](const size_t i) const { //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -QIBase::prime(const double x) const { +QIView::prime(const double x) const { const auto i0 = lowerBound(x); return mcoeffs[i0 + 1] + 2.0*mcoeffs[i0 + 2]*x; } SPHERAL_HOST_DEVICE inline double -QIBase::prime(const double x, +QIView::prime(const double x, const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return mcoeffs[i0 + 1] + 2.0*mcoeffs[i0 + 2]*x; @@ -84,14 +84,14 @@ QIBase::prime(const double x, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline double -QIBase::prime2(const double x) const { +QIView::prime2(const double x) const { const auto i0 = lowerBound(x); return 2.0*mcoeffs[i0 + 2]; } SPHERAL_HOST_DEVICE inline double -QIBase::prime2(const double /*x*/, +QIView::prime2(const double /*x*/, const size_t i0) const { REQUIRE(i0 <= 3u*mN1); return 2.0*mcoeffs[i0 + 2]; @@ -102,7 +102,7 @@ QIBase::prime2(const double /*x*/, //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline size_t -QIBase::lowerBound(const double x) const { +QIView::lowerBound(const double x) const { const auto result = 3u*std::min(mN1, size_t(std::max(0.0, x - mXmin)/mXstep)); ENSURE(result <= 3u*mN1); return result; @@ -113,25 +113,25 @@ QIBase::lowerBound(const double x) const { //------------------------------------------------------------------------------ SPHERAL_HOST_DEVICE inline size_t -QIBase::size() const { +QIView::size() const { return 3*(mN1 + 1u); } SPHERAL_HOST_DEVICE inline double -QIBase::xmin() const { +QIView::xmin() const { return mXmin; } SPHERAL_HOST_DEVICE inline double -QIBase::xmax() const { +QIView::xmax() const { return mXmax; } SPHERAL_HOST_DEVICE inline double -QIBase::xstep() const { +QIView::xstep() const { return mXstep; } @@ -141,7 +141,7 @@ QIBase::xstep() const { SPHERAL_HOST_DEVICE inline bool -QIBase::operator==(const QIBase& rhs) const { +QIView::operator==(const QIView& rhs) const { return ((mN1 == rhs.mN1) and (mXmin == rhs.mXmin) and (mXmax == rhs.mXmax) and diff --git a/tests/cpp/Utilities/chinterpolator_tests.cc b/tests/cpp/Utilities/chinterpolator_tests.cc index f15e0597e9..f943466c2f 100644 --- a/tests/cpp/Utilities/chinterpolator_tests.cc +++ b/tests/cpp/Utilities/chinterpolator_tests.cc @@ -45,9 +45,9 @@ GPU_TYPED_TEST_P(CubicHermiteInterpolatorTypedTest, CopyAssign) { SPHERAL_ASSERT_EQ(chi1.size(), chiref.size()); CHI chi2 = chiref; SPHERAL_ASSERT_EQ(chi2.size(), chiref.size()); - Spheral::CHIBase chi1_view = chi1.view(); - Spheral::CHIBase chi2_view = chi2.view(); - Spheral::CHIBase chiref_view = chiref; + Spheral::CHIView chi1_view = chi1.view(); + Spheral::CHIView chi2_view = chi2.view(); + Spheral::CHIView chiref_view = chiref; // Ensure the underlying data pointer is different from the initial CHI EXEC_IN_SPACE_BEGIN(TypeParam) SPHERAL_ASSERT_NE(chi1_view.data(), chiref_view.data()); @@ -69,7 +69,7 @@ GPU_TYPED_TEST_P(CubicHermiteInterpolatorTypedTest, FuncCtorTest) { CHI chih(xmin, xmax, NV, gpu_this->func); { size_t N = chih.size(); - Spheral::CHIBase chi = chih.view(); + Spheral::CHIView chi = chih.view(); EXEC_IN_SPACE_BEGIN(TypeParam) SPHERAL_ASSERT_EQ(chi.size(), N); EXEC_IN_SPACE_END() @@ -92,7 +92,7 @@ GPU_TYPED_TEST_P(CubicHermiteInterpolatorTypedTest, VecCtorTest) { const double xmax = gpu_this->xmax; CHI chih(xmin, xmax, yvals); size_t N = chih.size(); - Spheral::CHIBase chi = chih.view(); + Spheral::CHIView chi = chih.view(); EXEC_IN_SPACE_BEGIN(TypeParam) SPHERAL_ASSERT_EQ(chi.size(), N); EXEC_IN_SPACE_END() diff --git a/tests/cpp/Utilities/quadraticinterpolator_tests.cc b/tests/cpp/Utilities/quadraticinterpolator_tests.cc index 0fd746be6d..1740d70858 100644 --- a/tests/cpp/Utilities/quadraticinterpolator_tests.cc +++ b/tests/cpp/Utilities/quadraticinterpolator_tests.cc @@ -45,9 +45,9 @@ GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, CopyAssign) { SPHERAL_ASSERT_EQ(qi1.size(), qiref.size()); QI qi2 = qiref; SPHERAL_ASSERT_EQ(qi2.size(), qiref.size()); - Spheral::QIBase qi1_view = qi1.view(); - Spheral::QIBase qi2_view = qi2.view(); - Spheral::QIBase qiref_view = qiref; + Spheral::QIView qi1_view = qi1.view(); + Spheral::QIView qi2_view = qi2.view(); + Spheral::QIView qiref_view = qiref; // Ensure the underlying data pointer is different from the initial QI EXEC_IN_SPACE_BEGIN(TypeParam) SPHERAL_ASSERT_NE(qi1_view.data(), qiref_view.data()); @@ -69,7 +69,7 @@ GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, FuncCtorTest) { QI qih(xmin, xmax, NV, gpu_this->func); { size_t N = qih.size(); - Spheral::QIBase qi = qih.view(); + Spheral::QIView qi = qih.view(); EXEC_IN_SPACE_BEGIN(TypeParam) SPHERAL_ASSERT_EQ(qi.size(), N); EXEC_IN_SPACE_END() @@ -92,7 +92,7 @@ GPU_TYPED_TEST_P(QuadraticInterpolatorTypedTest, VecCtorTest) { const double xmax = gpu_this->xmax; QI qih(xmin, xmax, yvals); size_t N = qih.size(); - Spheral::QIBase qi = qih.view(); + Spheral::QIView qi = qih.view(); EXEC_IN_SPACE_BEGIN(TypeParam) SPHERAL_ASSERT_EQ(qi.size(), N); EXEC_IN_SPACE_END() From b7191e3af59e5a12e3585fe8f99ee760132cd354 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 27 Aug 2025 10:16:25 -0700 Subject: [PATCH 11/52] Removed unused line in QI --- extern/chai | 2 +- src/Utilities/QuadraticInterpolator.hh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/extern/chai b/extern/chai index 9bb1281f5c..df410ff295 160000 --- a/extern/chai +++ b/extern/chai @@ -1 +1 @@ -Subproject commit 9bb1281f5ceb8b39ff4c98d83c1fd80c5d814fcb +Subproject commit df410ff2959cd519512d0190d74fcbc77ee6fa47 diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index e06abf0ab3..1569fdfcfa 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -72,7 +72,6 @@ protected: class QuadraticInterpolator : public QIView { public: - using ContainerType = typename chai::ManagedArray; template QuadraticInterpolator(double xmin, double xmax, size_t n, const Func& F); QuadraticInterpolator(double xmin, double xmax, const std::vector& yvals); From 7420d98174ea12487874ad82dff398ca08f200c9 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 27 Aug 2025 15:40:17 -0700 Subject: [PATCH 12/52] Removed chai submodule, added package recipes for chai, raja, umpire, and camp, changed cmake variables to only use external chai --- .gitmodules | 5 ----- cmake/InstallTPLs.cmake | 22 ++++++---------------- cmake/spheral_cxx-config.cmake.in | 8 -------- extern/chai | 1 - scripts/spack/packages/camp/package.py | 13 +++++++++++++ scripts/spack/packages/chai/package.py | 15 +++++++++++++++ scripts/spack/packages/raja/package.py | 14 ++++++++++++++ scripts/spack/packages/spheral/package.py | 8 +++++--- scripts/spack/packages/umpire/package.py | 14 ++++++++++++++ 9 files changed, 67 insertions(+), 33 deletions(-) delete mode 160000 extern/chai create mode 100644 scripts/spack/packages/camp/package.py create mode 100644 scripts/spack/packages/chai/package.py create mode 100644 scripts/spack/packages/raja/package.py create mode 100644 scripts/spack/packages/umpire/package.py diff --git a/.gitmodules b/.gitmodules index e8b62da609..88a5b8e7f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,11 +8,6 @@ [submodule "extern/PolyClipper"] path = extern/PolyClipper url = https://github.com/LLNL/PolyClipper -[submodule "extern/chai"] - path = extern/chai - url = https://github.com/llnl/chai - branch = feature/ManagedSharedPtr - ignore = all [submodule "extern/ATS"] path = extern/ATS url = https://github.com/LLNL/ATS diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index a3f5663349..b15c1e6f3c 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -170,25 +170,15 @@ endif() message("-----------------------------------------------------------------------------") # Chai -if(chai_DIR AND USE_EXTERNAL_CHAI) - find_package(chai REQUIRED NO_DEFAULT_PATH PATHS ${chai_DIR}) - if (chai_FOUND) - message("Found chai External Package.") - endif() - list(APPEND SPHERAL_FP_TPLS chai) - list(APPEND SPHERAL_FP_DIRS ${chai_DIR}) -else() - message("Using chai Submodule.") - set(chai_DIR "${SPHERAL_ROOT_DIR}/extern/chai") - set(CHAI_ENABLE_TESTS Off) - set(CHAI_ENABLE_EXAMPLES Off) - set(CHAI_ENABLE_RAJA_PLUGIN On CACHE BOOL "") - add_subdirectory(${chai_DIR}) +find_package(chai REQUIRED NO_DEFAULT_PATH PATHS ${chai_DIR}) +if(chai_FOUND) + message("Found chai External Package.") + blt_convert_to_system_includes(TARGET chai) endif() list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire) -list(APPEND SPHERAL_FP_TPLS RAJA umpire) -list(APPEND SPHERAL_FP_DIRS ${raja_DIR} ${umpire_DIR}) +list(APPEND SPHERAL_FP_TPLS chai RAJA umpire) +list(APPEND SPHERAL_FP_DIRS ${chai_DIR} ${raja_DIR} ${umpire_DIR}) set_property(GLOBAL PROPERTY SPHERAL_FP_TPLS ${SPHERAL_FP_TPLS}) set_property(GLOBAL PROPERTY SPHERAL_FP_DIRS ${SPHERAL_FP_DIRS}) diff --git a/cmake/spheral_cxx-config.cmake.in b/cmake/spheral_cxx-config.cmake.in index 2781c48952..9493ae3068 100644 --- a/cmake/spheral_cxx-config.cmake.in +++ b/cmake/spheral_cxx-config.cmake.in @@ -31,14 +31,6 @@ if(NOT SPHERAL_FOUND) find_package(${tpl} REQUIRED QUIET NO_DEFAULT_PATH PATHS ${dir}) endif() endforeach() - if(NOT TARGET chai) - if (@USE_EXTERNAL_CHAI@) - set(SPHERAL_CHAI_DIR "@chai_DIR@/lib/cmake/chai") - else() - set(SPHERAL_CHAI_DIR "${SPHERAL_CXX_INSTALL_PREFIX}/lib/cmake/chai") - endif() - find_package(chai REQUIRED QUIET NO_DEFAULT_PATH PATHS ${SPHERAL_CHAI_DIR}) - endif() if(SPHERALC_STANDALONE) set(CMAKE_C_COMPILER "@CMAKE_C_COMPILER@" CACHE PATH "Spheral C compiler path") set(CMAKE_CXX_COMPILER "@CMAKE_CXX_COMPILER@" CACHE PATH "Spheral C++ compiler path") diff --git a/extern/chai b/extern/chai deleted file mode 160000 index 9bb1281f5c..0000000000 --- a/extern/chai +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9bb1281f5ceb8b39ff4c98d83c1fd80c5d814fcb diff --git a/scripts/spack/packages/camp/package.py b/scripts/spack/packages/camp/package.py new file mode 100644 index 0000000000..0e20eb4079 --- /dev/null +++ b/scripts/spack/packages/camp/package.py @@ -0,0 +1,13 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack.package import * +from spack.pkg.builtin.camp import Camp as BuiltinCamp + + +class Camp(BuiltinCamp): + + version("2025.03.0", tag="v2025.03.0", submodules=False) diff --git a/scripts/spack/packages/chai/package.py b/scripts/spack/packages/chai/package.py new file mode 100644 index 0000000000..7d7d6d7977 --- /dev/null +++ b/scripts/spack/packages/chai/package.py @@ -0,0 +1,15 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack.package import * +from spack.pkg.builtin.chai import Chai as BuiltinChai + + +class Chai(BuiltinChai): + + version("develop", commit="b7babdc0c333baa68e53b026c63d65c48c8d8eb1", submodules=False) + depends_on("raja@2025.03.2", type="build", when="+raja") + depends_on("umpire@2025.03.0", type="build") diff --git a/scripts/spack/packages/raja/package.py b/scripts/spack/packages/raja/package.py new file mode 100644 index 0000000000..d61e2b53f4 --- /dev/null +++ b/scripts/spack/packages/raja/package.py @@ -0,0 +1,14 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack.package import * +from spack.pkg.builtin.raja import Raja as BuiltinRaja + + +class Raja(BuiltinRaja): + + version("2025.03.2", tag="v2025.03.2", submodules=False) + depends_on("camp@2025.03.0", type="build") diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 3afe136818..6083d10a79 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -64,6 +64,8 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('silo +hdf5', type='build') + depends_on('chai@develop+raja', type='build') + depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') depends_on('axom@0.9.0 +hdf5 -lua -examples -python -fortran', type='build') @@ -78,8 +80,6 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('caliper+mpi', type='build', when='+mpi') depends_on('caliper~mpi', type='build', when='~mpi') - depends_on('raja@2024.02.0', type='build') - depends_on('opensubdiv@3.4.3+pic', type='build', when="+opensubdiv") depends_on('polytope +python', type='build', when="+python") @@ -96,7 +96,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on(f"{ctpl} {mpiv}", type='build', when=f"{mpiv}") # Forward CUDA/ROCM Variants - gpu_tpl_list = ["raja", "umpire", "axom"] + gpu_tpl_list = ["raja", "umpire", "axom", "chai"] for ctpl in gpu_tpl_list: for val in CudaPackage.cuda_arch_values: depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val}") @@ -235,6 +235,8 @@ def initconfig_package_entries(self): entries.append(cmake_cache_path('umpire_DIR', spec['umpire'].prefix)) + entries.append(cmake_cache_path('chai_DIR', spec['chai'].prefix)) + entries.append(cmake_cache_path('axom_DIR', spec['axom'].prefix)) entries.append(cmake_cache_path('silo_DIR', spec['silo'].prefix)) diff --git a/scripts/spack/packages/umpire/package.py b/scripts/spack/packages/umpire/package.py new file mode 100644 index 0000000000..e708c251c9 --- /dev/null +++ b/scripts/spack/packages/umpire/package.py @@ -0,0 +1,14 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack.package import * +from spack.pkg.builtin.umpire import Umpire as BuiltinUmpire + + +class Umpire(BuiltinUmpire): + + version("2025.03.0", tag="v2025.03.0", submodules=False) + depends_on("camp@2025.03.0", type="build") From f6302261b4ab9aee95a2f080da68a17863976360 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 27 Aug 2025 15:45:58 -0700 Subject: [PATCH 13/52] Chagned getPointer to data --- src/Field/FieldListInline.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Field/FieldListInline.hh b/src/Field/FieldListInline.hh index 796b53a9db..d5851e0c44 100644 --- a/src/Field/FieldListInline.hh +++ b/src/Field/FieldListInline.hh @@ -1920,7 +1920,7 @@ FieldListView FieldList::toView(FL&& extension, F&& field_extension) { auto callback = getFieldListCallback(std::forward(extension)); - if (mFieldViews.size() == 0 && !mFieldViews.getPointer(chai::CPU, false)) { + if (mFieldViews.size() == 0 && !mFieldViews.data(chai::CPU, false)) { mFieldViews.allocate(size(), chai::CPU, callback); } else { mFieldViews.setUserCallback(callback); From 0f30958982c41db30252d9f6cda402178e83c538 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 28 Aug 2025 09:46:41 -0700 Subject: [PATCH 14/52] Update release notes --- RELEASE_NOTES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 900c4faeb6..1f7132c128 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -34,6 +34,8 @@ Notable changes include: * Switched the CZ CI to use Dane instead of Ruby. * Increased the number of threads for certain memory intensive tests to prevent OOM error. * Updated GitHub actions since GitLab mirror changed. + * CHAI is no longer a submodule. + * CHAI, RAJA, Umpire, and Camp are all brought in through Spack as external TPLs now. Version v2025.06.1 -- Release date 2025-07-21 ============================================== From e3d703b9d294915f251cfd5cf5d32fa19a924d07 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 4 Sep 2025 10:05:06 -0700 Subject: [PATCH 15/52] Changed ENABLE_HIP to SPHERAL_ENABLE_HIP and added it to the export target script, updated leos version to 8.5.2, fixed leos spack package recipe, update spheral spack recipe, added link library option to SpheralHandleTPL.cmake --- cmake/SetupSpheral.cmake | 5 +- cmake/SpheralMacros.cmake | 36 +- cmake/spheral/SpheralHandleTPL.cmake | 3 + cmake/spheral_cxx-config.cmake.in | 1 + cmake/tpl/leos.cmake | 8 +- scripts/spack/packages/leos/package.py | 391 +++++------------- .../leos/patches/leos-8.5-umpire-import.patch | 45 ++ scripts/spack/packages/spheral/package.py | 26 +- 8 files changed, 181 insertions(+), 334 deletions(-) create mode 100644 scripts/spack/packages/leos/patches/leos-8.5-umpire-import.patch diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake index 2bdb1c6696..714d5d524b 100644 --- a/cmake/SetupSpheral.cmake +++ b/cmake/SetupSpheral.cmake @@ -69,6 +69,7 @@ option(SPHERAL_ENABLE_GSPH "Enable the GSPH package" ON) option(SPHERAL_ENABLE_SVPH "Enable the SVPH package" ON) option(SPHERAL_ENABLE_GLOBALDT_REDUCTION "Enable global allreduce for the time step" ON) option(SPHERAL_ENABLE_LONGCSDT "Enable longitudinal sound speed time step constraint" ON) +option(SPHERAL_ENABLE_HIP "Enable HIP" OFF) option(SPHERAL_ENABLE_LOGGER "Enable debug log printing" OFF) option(ENABLE_DEV_BUILD "Build separate internal C++ libraries for faster code development" OFF) @@ -97,10 +98,10 @@ if(ENABLE_CUDA) set(SPHERAL_ENABLE_CUDA ON) endif() -if(ENABLE_HIP) +if(SPHERAL_ENABLE_HIP) + set(ENABLE_HIP ON) list(APPEND SPHERAL_CXX_DEPENDS blt::hip) list(APPEND SPHERAL_CXX_DEPENDS blt::hip_runtime) - set(SPHERAL_ENABLE_HIP ON) endif() #-------------------------------------------------------------------------------# diff --git a/cmake/SpheralMacros.cmake b/cmake/SpheralMacros.cmake index bfaeff7164..3de46f45c4 100644 --- a/cmake/SpheralMacros.cmake +++ b/cmake/SpheralMacros.cmake @@ -7,26 +7,26 @@ macro(spheral_add_executable) "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN}) if (ENABLE_OPENMP) - list (APPEND arg_DEPENDS_ON openmp) - endif () + list(APPEND arg_DEPENDS_ON openmp) + endif() if (ENABLE_CUDA) - list (APPEND arg_DEPENDS_ON cuda) - endif () - - if (ENABLE_HIP) - list (APPEND arg_DEPENDS_ON blt::hip) - list (APPEND arg_DEPENDS_ON blt::hip_runtime) - endif () - - if (${arg_TEST}) - set (_output_dir ${CMAKE_BINARY_DIR}/tests) - elseif (${arg_REPRODUCER}) - set (_output_dir ${CMAKE_BINARY_DIR}/reproducers) - elseif (${arg_BENCHMARK}) - set (_output_dir ${CMAKE_BINARY_DIR}/benchmark) - else () - set (_output_dir ${CMAKE_BINARY_DIR}/bin) + list(APPEND arg_DEPENDS_ON cuda) + endif() + + if (SPHERAL_ENABLE_HIP) + list(APPEND arg_DEPENDS_ON blt::hip) + list(APPEND arg_DEPENDS_ON blt::hip_runtime) + endif() + + if(${arg_TEST}) + set(_output_dir ${CMAKE_BINARY_DIR}/tests) + elseif(${arg_REPRODUCER}) + set(_output_dir ${CMAKE_BINARY_DIR}/reproducers) + elseif(${arg_BENCHMARK}) + set(_output_dir ${CMAKE_BINARY_DIR}/benchmark) + else() + set(_output_dir ${CMAKE_BINARY_DIR}/bin) endif() blt_add_executable( diff --git a/cmake/spheral/SpheralHandleTPL.cmake b/cmake/spheral/SpheralHandleTPL.cmake index 3e75a8978b..3ad16759ed 100644 --- a/cmake/spheral/SpheralHandleTPL.cmake +++ b/cmake/spheral/SpheralHandleTPL.cmake @@ -86,6 +86,9 @@ function(Spheral_Handle_TPL lib_name TPL_CMAKE_DIR) INCLUDES ${${lib_name}_INCLUDE_DIR} LIBRARIES ${${lib_name}_LIBRARIES} EXPORTABLE ON) + if(${lib_name}_LINK_FLAGS) + target_link_options(${lib_name} INTERFACE ${${lib_name}_LINK_FLAGS}) + endif() get_target_property(_is_imported ${lib_name} IMPORTED) if(NOT ${_is_imported}) install(TARGETS ${lib_name} diff --git a/cmake/spheral_cxx-config.cmake.in b/cmake/spheral_cxx-config.cmake.in index 9493ae3068..29a1385495 100644 --- a/cmake/spheral_cxx-config.cmake.in +++ b/cmake/spheral_cxx-config.cmake.in @@ -10,6 +10,7 @@ if(NOT SPHERAL_FOUND) set(SPHERAL_ENABLE_MPI "@ENABLE_MPI@") set(SPHERAL_ENABLE_OPENMP "@ENABLE_OPENMP@") set(SPHERAL_ENABLE_CUDA "@ENABLE_CUDA@") + set(SPHERAL_ENABLE_HIP "@SPHERAL_ENABLE_HIP@") if(NOT axom_DIR) set(axom_DIR "@axom_DIR@" CACHE PATH "") endif() diff --git a/cmake/tpl/leos.cmake b/cmake/tpl/leos.cmake index e610b3b68c..e1243466de 100644 --- a/cmake/tpl/leos.cmake +++ b/cmake/tpl/leos.cmake @@ -1,7 +1 @@ -set(leos_libs libleos_C.a libleos.a liblip-cpp.a libyaml-cpp.a libleospact.a) -# If we ever support debug TPL builds we'll need the following since the name of the libarary changes -# if (CMAKE_BUILD_TYPE STREQUAL "Debug") -# list(APPEND leos_libs libyaml-cppd.a) -# else() -# list(APPEND leos_libs libyaml-cpp.a) -# endif() +set(leos_libs libleos.a liblip-cpp.a) diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index 6601904d4b..46af729b09 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -1,311 +1,112 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install leos +# +# You can always get back here to change things with: +# +# spack edit leos +# +# See the spack documentation for more information on building +# packages. +# from spack.package import * import os, re -import llnl.util.tty as tty -class Leos(CMakePackage, CudaPackage, ROCmPackage): - """This is derived from llnl.wci.legacy Leos package, but makes silo optional - and excludes ascmemory. - """ +class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): + """FIXME: put a proper description of your package here.""" + # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.example.com" fileLoc = '/usr/gapps/leos/srcs' fileUrl = 'file://' + fileLoc - version("8.4.1", sha256="93abeea9e336e3a81cc6cc9de10b2a2fd61eda2a89abece50cac80fef58ec38b", - url=os.path.join(fileUrl, "leos-8.4.1.tar.gz")) - version("8.4.2", sha256="08eb87580e30d7a1db72b1e1a457652dda9535df1c0caf7b5badb9cadf39f2a9", - url=os.path.join(fileUrl, "leos-8.4.2.tar.gz")) - - variant('debug', default=False, description='Build debug code (-g -O0)') - variant('filters', default=True , description='Build LEOS filter coding') - variant('raja', default=False, description='Build LIP using RAJA') - variant('cuda', default=False, description='Build LIP using RAJA + CUDA') - variant('umpire', default=False, description='Build LIP using UMPIRE') - variant('silo', default=True, description='Use Silo instead of LEOSPACT') - variant("cpp14", default=True, description="Sets the C++ std to C++14") - variant("yaml", default=True, description="Enable use of YAML") - - depends_on("mpi") - depends_on("hdf5+hl") + url = os.path.join(fileUrl, "leos-8.4.1.tar.gz") + + version("8.5.2", sha256="0fd104fd8599c5349d5156a433df0aa04880c01eb0105c9318493fc17b3b5a6f", preferred=True) + version("8.5.1", sha256="a072e48100bca21a594c6725158a0a7128f65ee4ce2aaa0be6e8fe55d3eff96a") + version("8.5.0", sha256="49b6549ce5fbca8afdd58f2266591f6ce68341b2f37bf4302c08c217a353362a") + version("8.4.2.2", sha256="9ffdd60995fa2aecc408c71ec7bd0e52eae0803bc372ff43389bcb5a1e59bbe0") + version("8.4.2.1", sha256="589a2b664e1bc1d245816dd536c950065ec2a6dac3f16b9ed53fb86e4e79a4db") + version("8.4.2", sha256="08eb87580e30d7a1db72b1e1a457652dda9535df1c0caf7b5badb9cadf39f2a9") + version("8.4.1", sha256="93abeea9e336e3a81cc6cc9de10b2a2fd61eda2a89abece50cac80fef58ec38b") + version("8.4.0", sha256="233333d0ac1bd0fa3a4eb756248c6c996b98bccb8dd957d9fac9e744fb6ede6b") + version("8.3.5", sha256="60d8298a5fc0dc056f33b39f664aab5ef75b4c4a4b3e1b80b22d769b39175db8") + version("8.3.3", sha256="ab54cba133f96bd52f82b84ea4e5a62c858b1e1e8a68ec3966e92183bffacaf3") + version("8.3.2", sha256="7f1c93404ccd1e39bef830632bf0e2f96348b9a75df19b1837594eb08729b846") + version("8.3.1", sha256="35ae5a24185e29111886adaee66628f9e0b6ed3198e8c6381ef3c53bf662fd55") + version("8.3.0", sha256="461fb0dc0672d5f284e392a8b70d9d50a035817aacb85a6843a5a75202a86cb5") + + variant("mpi", default=True, description="Build wit MPI enabled") + variant("debug", default=False, description="Build debug code (-g -O0)") + variant("filters", default=True , description="Build LEOS filter coding") + variant("lto", default=False, description="Build w/-dlto when cuda-11") + variant("cuda", default=False, description="Build LIP using RAJA + CUDA GPU code") + variant("rocm", default=False, description="Build LIP using RAJA + ROCM GPU code") + variant("silo", default=True, description="Use Silo instead of LEOSPACT") + + # FIXME: Add dependencies if this package requires them. + depends_on("mpi", when="+mpi") + depends_on("hdf5") depends_on("silo", when="+silo") depends_on("boost") depends_on("cmake") - depends_on("raja", when="+raja") + depends_on("zlib") depends_on("raja+cuda", when="+cuda") - depends_on("cnmem", when="+cuda @:8.2.5") + depends_on("raja+rocm", when="+rocm") depends_on("umpire+cuda", when="+cuda") - depends_on("umpire", when="+umpire") - depends_on("blt", type="build") - depends_on("camp") - - patch('patches/leos-8.2.0-CRAY.patch', when='@8.0.5:8.2.0 %cce') - patch('patches/leos-8.4.1-umpire-import.patch', when='@8.4.1') - patch('patches/leos-8.4.1-umpire-2024.02-exceptions.patch', when='@8.4.1 ^umpire@2024.02.0:') + depends_on("umpire+rocm", when="+rocm") + depends_on("camp", when="+umpire ^umpire@2022:") + depends_on("camp+cuda", when="+cuda ^umpire@2022:") - fpic_compilers = ['intel@14.toss', - 'intel@16.toss', - 'intel@17.toss', - 'intel@18.toss', - 'intel@19.toss', - 'intel@toss', - ] - - c_o2_compilers = ['pgi', - ] + patch("patches/leos-8.5-umpire-import.patch", when="@8.5+rocm") @property - def libs(self): - libnames = ['leos', 'leos_F', 'leos_C', 'lip-cpp'] - if (self.spec.satisfies('+yaml')): - libnames.append('yaml-cpp') - if (self.spec.satisfies('~silo')): - libnames.append('leospact') - libs = ['lib' + x for x in libnames] - return find_libraries(libs, self.spec.prefix, shared=False, recursive=True) + def cxx_std(self): + if "-std=c++20" in self.spec.compiler_flags["cxxflags"]: + return "20" + elif "-std=c++17" in self.spec.compiler_flags["cxxflags"]: + return "17" + elif "-std=c++14" in self.spec.compiler_flags["cxxflags"] or self.spec.satisfies("^umpire@2022:"): + return "14" + else: + return "11" - def setup_build_environment(self, env): + def initconfig_hardware_entries(self): spec = self.spec - if '+cuda' in spec: - env.set('CUDAHOSTCXX', spack_cxx) - - def cmake_args(self): + entries = super(Leos, self).initconfig_hardware_entries() + if spec.satisfies('+rocm'): + entries.append(cmake_cache_path("RAJA_DIR", spec["raja"].prefix)) + entries.append(cmake_cache_option("ENABLE_RAJA", True)) + entries.append(cmake_cache_path("UMPIRE_DIR", spec["umpire"].prefix)) + entries.append(cmake_cache_option("ENABLE_UMPIRE", True)) + entries.append(cmake_cache_option("ENABLE_RAJA_HIP", True)) + entries.append(cmake_cache_option("ENABLE_HIP", True)) + entries.append(cmake_cache_path("HIP_ROOT_DIR", spec["hip"].prefix)) + entries.append(cmake_cache_option("ENABLE_SHARED_MEMORY", False)) + entries.append(cmake_cache_option("ENABLE_ASCMEMORY", False)) + entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix)) + return entries + + def initconfig_package_entries(self): spec = self.spec - args = [] - - if spec.satisfies('+silo'): - args.extend([ - '-DSILO=ON', - "-DSILO_PATH=%s" % spec['silo'].prefix, - "-DUSE_PDB=ON", - ]) - else: - args.append('-DSILO=OFF') - - if '+cuda' in spec: - cuda_flags = None - if '%xl' in spec: - cuda_flags = '--expt-extended-lambda -Xptxas -warn-lmem-usage,-warn-spills -DCAMP_USE_PLATFORM_DEFAULT_STREAM' - elif '%gcc' in spec: - # Note: %gcc+cuda hasn't been tried yet - cuda_flags = '--expt-extended-lambda' - args.extend([ - '-DENABLE_CUDA:BOOL=ON', - '-DCMAKE_CUDA_COMPILER=%s/nvcc' % spec['cuda'].prefix.bin, - '-DCMAKE_CUDA_STANDARD=11', - '-DCMAKE_CUDA_SEPARABLE_COMPILATION=ON', - # Note this CMake option name for setting CUDA is nonstandard - '-DCUDA_ARCH=sm_%s' % spec.variants['cuda_arch'].value, - '-DCMAKE_CUDA_ARCHITECTURES=%s' % spec.variants['cuda_arch'].value, - ]) - if cuda_flags: - args.append('-DCMAKE_CUDA_FLAGS={0}'.format(cuda_flags)) - - if '+rocm' in spec: - args.extend([ - "-DENABLE_HIP=ON", - '-DHIP_CXX_COMPILER=' + str(self.compiler.cxx), - ]) - - cxx_flags = [] - if spec.satisfies('+raja'): - args.extend([ - "-DENABLE_RAJA=ON", - "-DRAJA_DIR=%s" % spec['raja'].prefix, - "-Dcamp_DIR={}/lib/cmake/camp".format(spec["camp"].prefix) - ]) - if spec.satisfies('+cuda'): - args.append('-DENABLE_RAJA_CUDA=ON') - cxx_flags.append(spec['cuda'].libs.ld_flags) - - if '+rocm' in spec: - # NOTE: This should not be required: FindRaja.cmake and raja-config.cmake - # should arrange for camp include directory. However, for some reason - # these files, in conjunction with cmake, are not performing the - # transitive search - hipcc_flags = ['-std=c++14'] - hipcc_flags.append('-I{0}'.format(spec['camp'].prefix.include)) - if 'camp+rocm' in spec: - hipcc_flags.append('-DCAMP_HAVE_HIP') - args.append('-DHIP_HIPCC_FLAGS={0}'.format(' '.join(hipcc_flags))) - - if cxx_flags: - args.append('-DCMAKE_CXX_FLAGS=' + ' '.join(cxx_flags)) - - if spec.satisfies('+umpire'): - umpire_libs = ';'.join(spec['umpire'].libs.libraries) - args.extend([ - "-DENABLE_UMPIRE=ON", - "-DENABLE_UMPIRE_IDS=ON", - "-DUMPIRE_DIR=%s" % spec['umpire'].prefix, - '-DUMPIRE_LIBRARIES=%s' % umpire_libs, - "-Dcamp_DIR={}/lib/cmake/camp".format(spec["camp"].prefix) - ]) - else: - args.extend([ - '-DENABLE_UMPIRE=OFF', - '-DENABLE_UMPIRE_IDS=OFF' - ]) - - if spec.satisfies('+filters'): - args.extend([ - "-DENABLE_FILTERS=ON", - "-DUSE_LIBXML2=OFF", - "-DUSE_NOXML=ON", - "-DFILTER_PLUGIN_DIR={0}".format(spec.prefix.lib) - ]) - else: - args.extend([ - "-DENABLE_FILTERS=OFF", - "-DUSE_LIBXML2=OFF", - "-DUSE_NOXML=ON" - ]) - if spec.satisfies('+yaml'): - args.append("-DENABLE_YAML:BOOL=ON") - else: - args.append("-DENABLE_YAML:BOOL=OFF") - - args += [ - '-DBLT_SOURCE_DIR={0}'.format(spec['blt'].prefix), - - '-DLEOS_BUILD_STANDALONE=OFF', - - '-DENABLE_MPI=ON', - '-DUSE_MPI=ON', - '-DMPI_C_COMPILER=%s' % self.spec['mpi'].mpicc, - '-DMPI_CXX_COMPILER=%s' % self.spec['mpi'].mpicxx, - '-DMPI_Fortran_COMPILER=%s' % self.spec['mpi'].mpifc, - - '-DENABLE_PYTHON=OFF', - '-DENABLE_ZFP=OFF', - '-DENABLE_OPENMP=OFF', - '-DENABLE_CALIPER=OFF', - - '-DENABLE_TESTS=OFF', - '-DBUILD_TESTS=OFF', - "-DENABLE_GMOCK=OFF", - "-DENABLE_GTEST=OFF", - '-DBUILD_EXAMPLES=OFF', - '-DBUILD_EXAMPLES_P=OFF', - '-DENABLE_EXAMPLES=OFF', - '-DENABLE_PARALLEL_EXAMPLES=OFF', - - "-DBUILD_TOOLS=OFF", - "-DENABLE_TOOLS=OFF", - - "-DUSE_TCALCFILTER=ON", - "-DENABLE_V7=ON", - "-DSTATIC_FILTERS=ON", - "-DENABLE_COMPRESSION=OFF", - "-DENABLE_SM_LIP=OFF", - "-DENABLE_C_LIP=ON", - "-DENABLE_CPP_LIP=ON", - "-DLIP_SHARED_MEMORY=OFF", - "-DENABLE_ASCMEMORY=OFF", - "-DENABLE_FPE_HANDLING=OFF", - "-DENABLE_UNCRUSTIFY=OFF", - "-DENABLE_VALGRIND=OFF", - - "-DUSE_HDF=ON", - "-DHDF_ROOT=%s" % spec['hdf5'].prefix, - "-DHDF5_ROOT=%s" % spec['hdf5'].prefix, - - "-DENABLE_FORTRAN=ON", - "-DENABLE_FORTRAN_INTERFACE=ON", - "-DENABLE_C=ON", - "-DENABLE_C_INTERFACE=ON", - - '-DEOS_DATA_ROOT_DIR=/usr/gapps/data/eos', - ] - - if "+cpp14" in spec: - args.extend(["-DBLT_CXX_STD=c++14", "-DCMAKE_CXX_STANDARD=14"]) - - # If present, pass in language-specific flags to CMake - cppflags = " ".join(spec.compiler_flags["cppflags"]) - if cppflags: - # avoid always ending up with ' ' with no flags defined - cppflags += " " - cflags = cppflags + " ".join(spec.compiler_flags["cflags"]) - if cflags: - args.append(f'-DBLT_C_FLAGS="{cflags}"') - - cxxflags = cppflags + " ".join(spec.compiler_flags["cxxflags"]) - if cxxflags: - args.append(f'-DBLT_CXX_FLAGS="{cxxflags}"') - - fflags = " ".join(spec.compiler_flags["fflags"]) - if spec.satisfies("%cce"): - fflags += " -ef" - if spec.satisfies("%rocmcc"): - fflags += " -O3 -Mstandard -Mfreeform -Mextend -Mbackslash -fno-default-real-8 -cpp" - if fflags: - args.append(f'-DBLT_FORTRAN_FLAGS={fflags}') - - return args - - def build(self, spec, prefix): - fflags_path = join_path(self.build_directory, 'interfaces/fortran/CMakeFiles/leos_F.dir/flags.make') - tty.msg("Fortran flags path: " + fflags_path) - filter_file('-DAND', '', fflags_path) - super(Leos, self).install(spec, prefix) - - def patch(self): - if self.spec.satisfies('@8.4.1: ^raja@2024.02.2:'): - filter_file('loop_', 'seq_', "LIP/cpp_source/raja_api/SimpleRajaDefinitions.hxx", backup=True) - - if self.spec.satisfies('@8.2.0') and self.spec.satisfies('%gcc@10:'): - filter_file(r'functional>', 'functional>\n#include', 'src/implementation/LEOS_MemoryManager.hxx') - filter_file(r'LEOS_Function_getMetadata_sS_len', 'LEOS_Function_getMetadata_sZ_len', 'interfaces/fortran/LEOS_Function_F.f90') - filter_file(r'LEOS_Function_getMetadata_sS_len', 'LEOS_Function_getMetadata_sZ_len', 'interfaces/C/LEOS_Function_C.h') - filter_file(r'LEOS_Function_getMetadata_sS_len', 'LEOS_Function_getMetadata_sZ_len', 'interfaces/C/LEOS_Function_C.cc') - elif self.spec.satisfies('@8.1.1p1') and ("%gcc" in self.spec or ("%clang" in self.spec and not self.spec.target.family=='ppc64le')): # gfortran issue - filter_file(r'LEOS_Function_getMetadata_sS_len', 'LEOS_Function_getMetadata_sZ_len', 'interfaces/fortran/LEOS_Function_F.f90') - filter_file(r'LEOS_Function_getMetadata_sS_len', 'LEOS_Function_getMetadata_sZ_len', 'interfaces/C/LEOS_Function_C.h') - filter_file(r'LEOS_Function_getMetadata_sS_len', 'LEOS_Function_getMetadata_sZ_len', 'interfaces/C/LEOS_Function_C.cc') - if self.spec.satisfies('@8.0.5:'): - filter_file(r'USE_MPI','USE_MPI AND BUILD_TESTS', 'interfaces/C/CMakeLists.txt') - filter_file(r'USE_MPI','USE_MPI AND BUILD_TESTS', 'interfaces/fortran/CMakeLists.txt') - filter_file(r'include\(leos-mpi\)','add_definitions(-DLEOS_USE_MPI)', 'CMakeLists.txt') - if self.spec.satisfies('@8.2:'): - filter_file(r'NOT \${HDF5_C_HL_LIBRARIES}', 'NOT "${HDF5_C_HL_LIBRARIES}"', 'cmake/Setup3rdParty.cmake') - - if "%pgi" in self.spec: - filter_file(r' tolower', ' ::tolower', 'src/io/ReaderFactory.cpp') - filter_file(r' -cpp','', 'interfaces/fortran/CMakeLists.txt') - if self.spec.satisfies('@8.2:'): - filter_file(r'__x86_64__', '__FOO__', 'cereal/include/cereal/external/rapidjson/internal/biginteger.h') - filter_file(r'__x86_64__', '__FOO__', 'cereal/include/cereal/external/rapidjson/internal/diyfp.h') # avoid uint128 undefined - - if "%clang@bgq" in self.spec or "%gcc@bgq" in self.spec: - filter_file(r'-WF,-D', '-D', 'interfaces/fortran/CMakeLists.txt') - elif self.spec.satisfies('@8.0.0:'): - if (self.spec.satisfies('platform=linux') and self.spec.target.family=='ppc64le') or ("%arm@sandia" in self.spec and self.spec.satisfies('@8.0.2:')): - filter_file(r'typedef int LEOS_MPI_Comm', '#include "mpi.h"\n typedef MPI_Comm LEOS_MPI_Comm', 'interfaces/C/LEOS_types_C.h') - - if self.spec.satisfies("@8.0.4:"): - datafile="src/io/ReaderFactory.cpp" - else: - datafile="src/io/databaseNames.h" - filter_file(r'/usr/gapps/data/eos', self.datadir, datafile) - if self.spec.satisfies('@8.0.5:8.1.0') and "%cce" in self.spec: - filter_file(r'\${TMP_VERS}','"#define BOOST_LIB_VERSION \"1_54\""', 'CMAKE/leos-boost.cmake') - if self.spec.satisfies('@8.0:8.2') and "%pgi" in self.spec: - filter_file(r'LIP_setup.h \*/', 'LIP_setup.h */\n#ifndef LIPSETUPH\n#define LIPSETUPH', 'LIP/source/LIP_setup.h') - with open("LIP/source/LIP_setup.h", 'a') as f: - f.write("#endif") - filter_file(r'LIP_proto.h \*/', 'LIP_proto.h */\n#ifndef LIPPROTOH\n#define LIPPROTOH', 'LIP/source/LIP_proto.h') - filter_file(r'\}\n', '}\n#endif\n', 'LIP/source/LIP_proto.h') - filter_file(r'LIP_setup.h', 'LIP_setup.h"\n#include "LIP_PBH.h', 'LIP/source/lip_setup_interp_shmem.cc') - - def old_flag_handler(self, name, flags): - my_flags = flags - if name in ('cflags', 'cxxflags'): - if any(self.spec.compiler.satisfies(c) for c in self.fpic_compilers): - my_flags.append(self.compiler.cc_pic_flag) - my_flags.append('-g -fp-model precise -fp-model source -unroll-aggressive -finline-functions -nolib-inline -axCORE-AVX2 -xAVX -no-fma -ip -no-ftz -prec-div -prec-sqrt -diag-disable cpu-dispatch') - else: - my_flags.append('-g') - if name == 'cxxflags' and any(self.spec.compiler.satisfies(c) for c in self.c_o2_compilers): - my_flags = list(map(lambda x: "" if x == "-O2" else x, my_flags)) - return (None, my_flags, None) - - @property - def datadir(self): - return '/usr/gapps/data/eos' + entries = [] + if spec.satisfies("+silo"): + entries.append(cmake_cache_option("ENABLE_SILO", True)) + entries.append(cmake_cache_path("SILO_PATH", spec["silo"].prefix)) + entries.append(cmake_cache_option("ENABLE_PDB", True)) + if spec.satisfies("+mpi"): + entries.append(cmake_cache_option("ENABLE_MPI", True)) + entries.append(cmake_cache_path('-DMPI_C_COMPILER', spec['mpi'].mpicc)) + entries.append(cmake_cache_path('-DMPI_CXX_COMPILER', spec['mpi'].mpicxx)) + features_disabled = ["PYTHON", "PYTHON_INTERFACE", "ZFP", "YAML", "TESTS", + "TOOLS", "EXAMPLES", "PARALLEL_EXAMPLES", "FORTRAN_INTERFACE"] + for fd in features_disabled: + entries.append(cmake_cache_option(f"ENABLE_{fd}", False)) + entries.append(cmake_cache_option("ENABLE_CPP_LIP", True)) + entries.append(cmake_cache_option("ENABLE_HDF", True)) + entries.append(cmake_cache_path("HDF5_ROOT", spec["hdf5"].prefix)) + entries.append(cmake_cache_path("EOS_DATA_ROOT_DIR", "/usr/gapps/data/eos")) + return entries diff --git a/scripts/spack/packages/leos/patches/leos-8.5-umpire-import.patch b/scripts/spack/packages/leos/patches/leos-8.5-umpire-import.patch new file mode 100644 index 0000000000..ff15190d99 --- /dev/null +++ b/scripts/spack/packages/leos/patches/leos-8.5-umpire-import.patch @@ -0,0 +1,45 @@ +diff -u -r leos-8.5.2/cmake/TPL_Setup/SetupUmpire.cmake newleos/cmake/TPL_Setup/SetupUmpire.cmake +--- leos-8.5.2/cmake/TPL_Setup/SetupUmpire.cmake 2025-07-03 04:04:38.000000000 -0700 ++++ newleos/cmake/TPL_Setup/SetupUmpire.cmake 2025-09-03 16:26:05.717203000 -0700 +@@ -5,26 +5,8 @@ + #--------------------------------------------------------------------- + # Setup paths to point to directory which can hold umpire-config.cmake + #--------------------------------------------------------------------- +-string(FIND ${UMPIRE_DIR} "lib/cmake/umpire" NEW_UMPIRE_CONFIG_LOC) +-string(FIND ${UMPIRE_DIR} "share/umpire/cmake" OLD_UMPIRE_CONFIG_LOC) +- +-if ( NEW_UMPIRE_CONFIG_LOC GREATER_EQUAL 0 ) +- string(REPLACE "/lib/cmake/umpire" "" BASE_UMPIRE_DIR ${UMPIRE_DIR} ) +-elseif ( OLD_UMPIRE_CONFIG_LOC GREATER_EQUAL 0 ) +- string(REPLACE "/share/umpire/cmake" "" BASE_UMPIRE_DIR ${UMPIRE_DIR} ) +-else() +- # Look for directory with umpire-config.cmake +- set ( BASE_UMPIRE_DIR ${UMPIRE_DIR} ) +- +- # Check for directory holding umpire-config.cmake +- if ( IS_DIRECTORY ${BASE_UMPIRE_DIR}/lib/cmake/umpire) +- set (UMPIRE_DIR ${BASE_UMPIRE_DIR}/lib/cmake/umpire) +- elseif(IS_DIRECTORY ${BASE_UMPIRE_DIR}/share/umpire/cmake) +- set (UMPIRE_DIR ${BASE_UMPIRE_DIR}/share/umpire/cmake) +- elseif(NOT EXISTS ${UMPIRE_DIR}/umpire-config.cmake) +- message(SEND_ERROR "Cannot find path to umpire-config.cmake") +- endif() +-endif() ++# Look for directory with umpire-config.cmake ++set ( BASE_UMPIRE_DIR ${UMPIRE_DIR} ) + + if( NOT DEFINED camp_DIR) + +@@ -46,7 +28,11 @@ + endif( NOT DEFINED camp_DIR) + + #set(UMPIRE_MIN_VERSION "5.0") +-find_package(UMPIRE ${UMPIRE_MIN_VERSION} REQUIRED) ++find_package(UMPIRE ${UMPIRE_MIN_VERSION} REQUIRED NO_DEFAULT_PATH ++ PATHS ${UMPIRE_DIR}/share/umpire/cmake ++ ${UMPIRE_DIR}/lib/cmake/umpire ++ ${UMPIRE_DIR}/lib64/cmake/umpire ++ ${UMPIRE_DIR}) + + if (UMPIRE_FOUND) + diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 6083d10a79..2438cba837 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -87,7 +87,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') - depends_on('leos@8.4.2', type='build', when='+leos') + depends_on('leos@8.5.0', type='build', when='+leos') # Forward MPI Variants mpi_tpl_list = ["hdf5", "conduit", "axom", "adiak~shared"] @@ -97,6 +97,8 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): # Forward CUDA/ROCM Variants gpu_tpl_list = ["raja", "umpire", "axom", "chai"] + if (LEOSpresent): + gpu_tpl_list.append("leos") for ctpl in gpu_tpl_list: for val in CudaPackage.cuda_arch_values: depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val}") @@ -165,7 +167,7 @@ def initconfig_compiler_entries(self): def initconfig_mpi_entries(self): spec = self.spec entries = [] - if "+mpi" in spec: + if spec.satisfied("+mpi"): entries = super(Spheral, self).initconfig_mpi_entries() # When on cray / flux systems we need to tell CMAKE the mpi flag explicitly if "cray-mpich" in spec: @@ -179,11 +181,11 @@ def initconfig_hardware_entries(self): spec = self.spec entries = super(Spheral, self).initconfig_hardware_entries() - if '+rocm' in spec: - entries.append(cmake_cache_option("ENABLE_HIP", True)) + if spec.satisfies('+rocm'): + entries.append(cmake_cache_option("SPHERAL_ENABLE_HIP", True)) entries.append(cmake_cache_string("ROCM_PATH", spec["hip"].prefix)) - if '+cuda' in spec: + if spec.satisfies('+cuda'): entries.append(cmake_cache_option("ENABLE_CUDA", True)) if not spec.satisfies('cuda_arch=none'): @@ -216,7 +218,7 @@ def initconfig_package_entries(self): entries.append(cmake_cache_string('SPHERAL_SPEC', self._get_short_spec(spec))) # TPL locations - if (spec.satisfies("+caliper")): + if spec.satisfies("+caliper"): entries.append(cmake_cache_path('caliper_DIR', spec['caliper'].prefix)) entries.append(cmake_cache_path('adiak_DIR', spec['adiak'].prefix)) @@ -244,27 +246,27 @@ def initconfig_package_entries(self): entries.append(cmake_cache_path('eigen_DIR', spec['eigen'].prefix)) entries.append(cmake_cache_path('eigen_INCLUDES',spec['eigen'].prefix.include.eigen3)) - if (spec.satisfies("+opensubdiv")): + if spec.satisfies("+opensubdiv"): entries.append(cmake_cache_path('opensubdiv_DIR', spec['opensubdiv'].prefix)) entries.append(cmake_cache_option('ENABLE_OPENSUBDIV', True)) - if (spec.satisfies("~network")): + if spec.satisfies("~network"): entries.append(cmake_cache_option('SPHERAL_NETWORK_CONNECTED', False)) entries.append(cmake_cache_path('polytope_DIR', spec['polytope'].prefix)) - entries.append(cmake_cache_option('ENABLE_MPI', '+mpi' in spec)) - if "+mpi" in spec: + if spec.satisfies("+mpi"): + entries.append(cmake_cache_option('ENABLE_MPI', True)) entries.append(cmake_cache_path('-DMPI_C_COMPILER', spec['mpi'].mpicc) ) entries.append(cmake_cache_path('-DMPI_CXX_COMPILER', spec['mpi'].mpicxx) ) - if "~shared" in spec and "~cuda" in spec: + if spec.satisfies("~shared") and spec.satisfies("~cuda"): entries.append(cmake_cache_option('ENABLE_SHARED', False)) entries.append(cmake_cache_option('ENABLE_OPENMP', '+openmp' in spec)) entries.append(cmake_cache_option('ENABLE_DOCS', '+docs' in spec)) - if "+python" in spec: + if spec.satisfies("+python"): entries.append(cmake_cache_path('python_DIR', spec['python'].prefix)) if spec.satisfies("+sundials"): From a21597de08b5eac60f4bfe165fcbf6da319bcbc9 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 4 Sep 2025 10:17:23 -0700 Subject: [PATCH 16/52] Removed umpire variant logic --- scripts/spack/packages/leos/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index 46af729b09..ac6f343b7d 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -58,8 +58,8 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on("raja+rocm", when="+rocm") depends_on("umpire+cuda", when="+cuda") depends_on("umpire+rocm", when="+rocm") - depends_on("camp", when="+umpire ^umpire@2022:") - depends_on("camp+cuda", when="+cuda ^umpire@2022:") + depends_on("camp+cuda", when="+cuda") + depends_on("camp+rocm", when="+rocm") patch("patches/leos-8.5-umpire-import.patch", when="@8.5+rocm") From afd2f5801d0eac8acbbc96ec19b75704b5292c1e Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 4 Sep 2025 10:55:46 -0700 Subject: [PATCH 17/52] Fix spack package typo --- scripts/spack/packages/spheral/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 2438cba837..6948789b42 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -167,7 +167,7 @@ def initconfig_compiler_entries(self): def initconfig_mpi_entries(self): spec = self.spec entries = [] - if spec.satisfied("+mpi"): + if spec.satisfies("+mpi"): entries = super(Spheral, self).initconfig_mpi_entries() # When on cray / flux systems we need to tell CMAKE the mpi flag explicitly if "cray-mpich" in spec: From 63c17704270be751ecd0db34ec2cf082a342d14e Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 4 Sep 2025 13:31:08 -0700 Subject: [PATCH 18/52] Turn off mpi for axom when ~mpi --- scripts/spack/packages/spheral/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 6948789b42..5e9dbb4991 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -69,6 +69,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') depends_on('axom@0.9.0 +hdf5 -lua -examples -python -fortran', type='build') + depends_on('axom+mpi', when='+mpi') with when('+rocm') or when('+cuda'): depends_on('axom ~shared', type='build') From f5b71f2aae6bc3263e9721ad2d2e86343ed006e6 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 4 Sep 2025 13:39:59 -0700 Subject: [PATCH 19/52] Change back to using ENABLE_HIP, let that be a different PR --- cmake/SetupSpheral.cmake | 5 ++--- cmake/SpheralMacros.cmake | 2 +- cmake/spheral_cxx-config.cmake.in | 2 +- scripts/spack/packages/spheral/package.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake index 714d5d524b..927c308eb8 100644 --- a/cmake/SetupSpheral.cmake +++ b/cmake/SetupSpheral.cmake @@ -69,7 +69,6 @@ option(SPHERAL_ENABLE_GSPH "Enable the GSPH package" ON) option(SPHERAL_ENABLE_SVPH "Enable the SVPH package" ON) option(SPHERAL_ENABLE_GLOBALDT_REDUCTION "Enable global allreduce for the time step" ON) option(SPHERAL_ENABLE_LONGCSDT "Enable longitudinal sound speed time step constraint" ON) -option(SPHERAL_ENABLE_HIP "Enable HIP" OFF) option(SPHERAL_ENABLE_LOGGER "Enable debug log printing" OFF) option(ENABLE_DEV_BUILD "Build separate internal C++ libraries for faster code development" OFF) @@ -98,8 +97,8 @@ if(ENABLE_CUDA) set(SPHERAL_ENABLE_CUDA ON) endif() -if(SPHERAL_ENABLE_HIP) - set(ENABLE_HIP ON) +if(ENABLE_HIP) + set(SPHERAL_ENABLE_HIP ON) list(APPEND SPHERAL_CXX_DEPENDS blt::hip) list(APPEND SPHERAL_CXX_DEPENDS blt::hip_runtime) endif() diff --git a/cmake/SpheralMacros.cmake b/cmake/SpheralMacros.cmake index 3de46f45c4..8438f20df9 100644 --- a/cmake/SpheralMacros.cmake +++ b/cmake/SpheralMacros.cmake @@ -14,7 +14,7 @@ macro(spheral_add_executable) list(APPEND arg_DEPENDS_ON cuda) endif() - if (SPHERAL_ENABLE_HIP) + if (ENABLE_HIP) list(APPEND arg_DEPENDS_ON blt::hip) list(APPEND arg_DEPENDS_ON blt::hip_runtime) endif() diff --git a/cmake/spheral_cxx-config.cmake.in b/cmake/spheral_cxx-config.cmake.in index 29a1385495..0ebd6dec1f 100644 --- a/cmake/spheral_cxx-config.cmake.in +++ b/cmake/spheral_cxx-config.cmake.in @@ -10,7 +10,7 @@ if(NOT SPHERAL_FOUND) set(SPHERAL_ENABLE_MPI "@ENABLE_MPI@") set(SPHERAL_ENABLE_OPENMP "@ENABLE_OPENMP@") set(SPHERAL_ENABLE_CUDA "@ENABLE_CUDA@") - set(SPHERAL_ENABLE_HIP "@SPHERAL_ENABLE_HIP@") + set(SPHERAL_ENABLE_HIP "@ENABLE_HIP@") if(NOT axom_DIR) set(axom_DIR "@axom_DIR@" CACHE PATH "") endif() diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 5e9dbb4991..a396ef2336 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -183,7 +183,7 @@ def initconfig_hardware_entries(self): entries = super(Spheral, self).initconfig_hardware_entries() if spec.satisfies('+rocm'): - entries.append(cmake_cache_option("SPHERAL_ENABLE_HIP", True)) + entries.append(cmake_cache_option("ENABLE_HIP", True)) entries.append(cmake_cache_string("ROCM_PATH", spec["hip"].prefix)) if spec.satisfies('+cuda'): From ed7355c90364272e0dc4697d91ef460e654cdd85 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 4 Sep 2025 15:19:33 -0700 Subject: [PATCH 20/52] Removed non-mpi CI tests --- .gitlab/jobs-seq.yml | 70 +++++++++----------------------------------- .gitlab/specs.yml | 21 +++---------- 2 files changed, 18 insertions(+), 73 deletions(-) diff --git a/.gitlab/jobs-seq.yml b/.gitlab/jobs-seq.yml index 0893d83500..418a30bfbf 100644 --- a/.gitlab/jobs-seq.yml +++ b/.gitlab/jobs-seq.yml @@ -1,63 +1,21 @@ # ------------------------------------------------------------------------------ # BUILD JOBS -toss_gcc_~mpi_init: - extends: [.gcc_~mpi, .init, .toss_shell1] +cray_rocm_Debug_init: + extends: [.cray_shell1, .rocm_Debug, .init] -toss_gcc_~mpi_tpls: - extends: [.gcc_~mpi, .tpls, .toss_resource1] - needs: [toss_gcc_~mpi_init] +cray_rocm_Debug_tpls: + extends: [.cray_resource1, .rocm_Debug, .tpls] + needs: [cray_rocm_Debug_init] -toss_gcc_~mpi_build: - extends: [.gcc_~mpi, .build_and_test, .toss_resource1] - needs: [toss_gcc_~mpi_tpls] +cray_rocm_Debug_build: + extends: [.cray_resource1, .rocm_Debug, .build_and_test] + needs: [cray_rocm_Debug_tpls] -toss_gcc_~mpi_test: - extends: [.gcc_~mpi, .run_ats, .toss_resource1] - needs: [toss_gcc_~mpi_build] +cray_rocm_Debug_test: + extends: [.cray_resource1, .rocm_Debug, .run_ats] + needs: [cray_rocm_Debug_build] -toss_gcc_~mpi_cleanup: - extends: [.gcc_~mpi, .cleanup_dir, .toss_resource1] - needs: [toss_gcc_~mpi_test] - - - -cray_hip_rocm_~mpi_init: - extends: [.cray_shell1, .hip_rocm_~mpi, .init] - -cray_hip_rocm_~mpi_tpls: - extends: [.cray_resource2, .hip_rocm_~mpi, .tpls] - needs: [cray_hip_rocm_~mpi_init] - -cray_hip_rocm_~mpi_build: - extends: [.cray_resource2, .hip_rocm_~mpi, .build_and_test] - needs: [cray_hip_rocm_~mpi_tpls] - -cray_hip_rocm_~mpi_test: - extends: [.cray_resource2, .hip_rocm_~mpi, .run_ats] - needs: [cray_hip_rocm_~mpi_build] - -cray_hip_rocm_~mpi_cleanup: - extends: [.cray_resource2, .hip_rocm_~mpi, .cleanup_dir] - needs: [cray_hip_rocm_~mpi_test] - - - -cray_rocm_~mpi_Debug_init: - extends: [.cray_shell1, .rocm_~mpi_Debug, .init] - -cray_rocm_~mpi_Debug_tpls: - extends: [.cray_resource1, .rocm_~mpi_Debug, .tpls] - needs: [cray_rocm_~mpi_Debug_init] - -cray_rocm_~mpi_Debug_build: - extends: [.cray_resource1, .rocm_~mpi_Debug, .build_and_test] - needs: [cray_rocm_~mpi_Debug_tpls] - -cray_rocm_~mpi_Debug_test: - extends: [.cray_resource1, .rocm_~mpi_Debug, .run_ats] - needs: [cray_rocm_~mpi_Debug_build] - -cray_rocm_~mpi_Debug_cleanup: - extends: [.cray_resource1, .rocm_~mpi_Debug, .cleanup_dir] - needs: [cray_rocm_~mpi_Debug_test] +cray_rocm_Debug_cleanup: + extends: [.cray_resource1, .rocm_Debug, .cleanup_dir] + needs: [cray_rocm_Debug_test] diff --git a/.gitlab/specs.yml b/.gitlab/specs.yml index 0ffe7db585..e362733744 100644 --- a/.gitlab/specs.yml +++ b/.gitlab/specs.yml @@ -11,14 +11,9 @@ SPEC: '%gcc+mpi' EXTRA_CMAKE_ARGS: '-DENABLE_DOCS=On -DENABLE_WARNINGS_AS_ERRORS=On' -.gcc_~mpi: +.gcc_Debug: variables: - SPEC: '%gcc~mpi' - EXTRA_CMAKE_ARGS: '-DENABLE_DOCS=On -DENABLE_WARNINGS_AS_ERRORS=On' - -.gcc_~mpi_Debug: - variables: - SPEC: '%gcc~mpi' + SPEC: '%gcc' EXTRA_CMAKE_ARGS: '-DCMAKE_BUILD_TYPE=Debug -DENABLE_WARNINGS_AS_ERRORS=On' @@ -44,19 +39,11 @@ variables: SPEC: '%rocmcc+mpi~rocm' -.rocm_~mpi: +.rocm_Debug: variables: - SPEC: '%rocmcc~mpi~rocm' - -.rocm_~mpi_Debug: - variables: - SPEC: '%rocmcc~mpi~rocm' + SPEC: '%rocmcc~rocm' EXTRA_CMAKE_ARGS: '-DCMAKE_BUILD_TYPE=Debug -DENABLE_WARNINGS_AS_ERRORS=On' .hip_rocm_mpich: variables: SPEC: '%rocmcc+mpi+rocm' - -.hip_rocm_~mpi: - variables: - SPEC: '%rocmcc~mpi+rocm' From a5d599de05acc21d6bf893d0f35cd29394e7f51c Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 4 Sep 2025 15:25:53 -0700 Subject: [PATCH 21/52] Removed redundant axom variant added a few commits ago, added spheral openmp define though it is unused --- cmake/SetupSpheral.cmake | 1 + scripts/spack/packages/spheral/package.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake index 927c308eb8..795bc48424 100644 --- a/cmake/SetupSpheral.cmake +++ b/cmake/SetupSpheral.cmake @@ -86,6 +86,7 @@ if(ENABLE_MPI) endif() if(ENABLE_OPENMP) + set(SPHERAL_ENABLE_OPENMP) list(APPEND SPHERAL_CXX_DEPENDS openmp) endif() diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index a396ef2336..5230f2b126 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -69,7 +69,6 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') depends_on('axom@0.9.0 +hdf5 -lua -examples -python -fortran', type='build') - depends_on('axom+mpi', when='+mpi') with when('+rocm') or when('+cuda'): depends_on('axom ~shared', type='build') From 68c45cad4317764ee5aa1d1af132ae5013db0c41 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 5 Sep 2025 09:47:55 -0700 Subject: [PATCH 22/52] Revert "Removed redundant axom variant added a few commits ago, added spheral openmp define though it is unused" This reverts commit a5d599de05acc21d6bf893d0f35cd29394e7f51c. --- cmake/SetupSpheral.cmake | 1 - scripts/spack/packages/spheral/package.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake index 795bc48424..927c308eb8 100644 --- a/cmake/SetupSpheral.cmake +++ b/cmake/SetupSpheral.cmake @@ -86,7 +86,6 @@ if(ENABLE_MPI) endif() if(ENABLE_OPENMP) - set(SPHERAL_ENABLE_OPENMP) list(APPEND SPHERAL_CXX_DEPENDS openmp) endif() diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 5230f2b126..a396ef2336 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -69,6 +69,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') depends_on('axom@0.9.0 +hdf5 -lua -examples -python -fortran', type='build') + depends_on('axom+mpi', when='+mpi') with when('+rocm') or when('+cuda'): depends_on('axom ~shared', type='build') From 918a2ad40c5a2b4c38da8bf7722404fb9b421bc8 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 5 Sep 2025 09:47:58 -0700 Subject: [PATCH 23/52] Revert "Removed non-mpi CI tests" This reverts commit ed7355c90364272e0dc4697d91ef460e654cdd85. --- .gitlab/jobs-seq.yml | 70 +++++++++++++++++++++++++++++++++++--------- .gitlab/specs.yml | 21 ++++++++++--- 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/.gitlab/jobs-seq.yml b/.gitlab/jobs-seq.yml index 418a30bfbf..0893d83500 100644 --- a/.gitlab/jobs-seq.yml +++ b/.gitlab/jobs-seq.yml @@ -1,21 +1,63 @@ # ------------------------------------------------------------------------------ # BUILD JOBS -cray_rocm_Debug_init: - extends: [.cray_shell1, .rocm_Debug, .init] +toss_gcc_~mpi_init: + extends: [.gcc_~mpi, .init, .toss_shell1] -cray_rocm_Debug_tpls: - extends: [.cray_resource1, .rocm_Debug, .tpls] - needs: [cray_rocm_Debug_init] +toss_gcc_~mpi_tpls: + extends: [.gcc_~mpi, .tpls, .toss_resource1] + needs: [toss_gcc_~mpi_init] -cray_rocm_Debug_build: - extends: [.cray_resource1, .rocm_Debug, .build_and_test] - needs: [cray_rocm_Debug_tpls] +toss_gcc_~mpi_build: + extends: [.gcc_~mpi, .build_and_test, .toss_resource1] + needs: [toss_gcc_~mpi_tpls] -cray_rocm_Debug_test: - extends: [.cray_resource1, .rocm_Debug, .run_ats] - needs: [cray_rocm_Debug_build] +toss_gcc_~mpi_test: + extends: [.gcc_~mpi, .run_ats, .toss_resource1] + needs: [toss_gcc_~mpi_build] -cray_rocm_Debug_cleanup: - extends: [.cray_resource1, .rocm_Debug, .cleanup_dir] - needs: [cray_rocm_Debug_test] +toss_gcc_~mpi_cleanup: + extends: [.gcc_~mpi, .cleanup_dir, .toss_resource1] + needs: [toss_gcc_~mpi_test] + + + +cray_hip_rocm_~mpi_init: + extends: [.cray_shell1, .hip_rocm_~mpi, .init] + +cray_hip_rocm_~mpi_tpls: + extends: [.cray_resource2, .hip_rocm_~mpi, .tpls] + needs: [cray_hip_rocm_~mpi_init] + +cray_hip_rocm_~mpi_build: + extends: [.cray_resource2, .hip_rocm_~mpi, .build_and_test] + needs: [cray_hip_rocm_~mpi_tpls] + +cray_hip_rocm_~mpi_test: + extends: [.cray_resource2, .hip_rocm_~mpi, .run_ats] + needs: [cray_hip_rocm_~mpi_build] + +cray_hip_rocm_~mpi_cleanup: + extends: [.cray_resource2, .hip_rocm_~mpi, .cleanup_dir] + needs: [cray_hip_rocm_~mpi_test] + + + +cray_rocm_~mpi_Debug_init: + extends: [.cray_shell1, .rocm_~mpi_Debug, .init] + +cray_rocm_~mpi_Debug_tpls: + extends: [.cray_resource1, .rocm_~mpi_Debug, .tpls] + needs: [cray_rocm_~mpi_Debug_init] + +cray_rocm_~mpi_Debug_build: + extends: [.cray_resource1, .rocm_~mpi_Debug, .build_and_test] + needs: [cray_rocm_~mpi_Debug_tpls] + +cray_rocm_~mpi_Debug_test: + extends: [.cray_resource1, .rocm_~mpi_Debug, .run_ats] + needs: [cray_rocm_~mpi_Debug_build] + +cray_rocm_~mpi_Debug_cleanup: + extends: [.cray_resource1, .rocm_~mpi_Debug, .cleanup_dir] + needs: [cray_rocm_~mpi_Debug_test] diff --git a/.gitlab/specs.yml b/.gitlab/specs.yml index e362733744..0ffe7db585 100644 --- a/.gitlab/specs.yml +++ b/.gitlab/specs.yml @@ -11,9 +11,14 @@ SPEC: '%gcc+mpi' EXTRA_CMAKE_ARGS: '-DENABLE_DOCS=On -DENABLE_WARNINGS_AS_ERRORS=On' -.gcc_Debug: +.gcc_~mpi: variables: - SPEC: '%gcc' + SPEC: '%gcc~mpi' + EXTRA_CMAKE_ARGS: '-DENABLE_DOCS=On -DENABLE_WARNINGS_AS_ERRORS=On' + +.gcc_~mpi_Debug: + variables: + SPEC: '%gcc~mpi' EXTRA_CMAKE_ARGS: '-DCMAKE_BUILD_TYPE=Debug -DENABLE_WARNINGS_AS_ERRORS=On' @@ -39,11 +44,19 @@ variables: SPEC: '%rocmcc+mpi~rocm' -.rocm_Debug: +.rocm_~mpi: variables: - SPEC: '%rocmcc~rocm' + SPEC: '%rocmcc~mpi~rocm' + +.rocm_~mpi_Debug: + variables: + SPEC: '%rocmcc~mpi~rocm' EXTRA_CMAKE_ARGS: '-DCMAKE_BUILD_TYPE=Debug -DENABLE_WARNINGS_AS_ERRORS=On' .hip_rocm_mpich: variables: SPEC: '%rocmcc+mpi+rocm' + +.hip_rocm_~mpi: + variables: + SPEC: '%rocmcc~mpi+rocm' From b6c136fdd3148d44ceca52783910132b8fd24a0c Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 5 Sep 2025 09:48:02 -0700 Subject: [PATCH 24/52] Revert "Change back to using ENABLE_HIP, let that be a different PR" This reverts commit f5b71f2aae6bc3263e9721ad2d2e86343ed006e6. --- cmake/SetupSpheral.cmake | 5 +++-- cmake/SpheralMacros.cmake | 2 +- cmake/spheral_cxx-config.cmake.in | 2 +- scripts/spack/packages/spheral/package.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake index 927c308eb8..714d5d524b 100644 --- a/cmake/SetupSpheral.cmake +++ b/cmake/SetupSpheral.cmake @@ -69,6 +69,7 @@ option(SPHERAL_ENABLE_GSPH "Enable the GSPH package" ON) option(SPHERAL_ENABLE_SVPH "Enable the SVPH package" ON) option(SPHERAL_ENABLE_GLOBALDT_REDUCTION "Enable global allreduce for the time step" ON) option(SPHERAL_ENABLE_LONGCSDT "Enable longitudinal sound speed time step constraint" ON) +option(SPHERAL_ENABLE_HIP "Enable HIP" OFF) option(SPHERAL_ENABLE_LOGGER "Enable debug log printing" OFF) option(ENABLE_DEV_BUILD "Build separate internal C++ libraries for faster code development" OFF) @@ -97,8 +98,8 @@ if(ENABLE_CUDA) set(SPHERAL_ENABLE_CUDA ON) endif() -if(ENABLE_HIP) - set(SPHERAL_ENABLE_HIP ON) +if(SPHERAL_ENABLE_HIP) + set(ENABLE_HIP ON) list(APPEND SPHERAL_CXX_DEPENDS blt::hip) list(APPEND SPHERAL_CXX_DEPENDS blt::hip_runtime) endif() diff --git a/cmake/SpheralMacros.cmake b/cmake/SpheralMacros.cmake index 8438f20df9..3de46f45c4 100644 --- a/cmake/SpheralMacros.cmake +++ b/cmake/SpheralMacros.cmake @@ -14,7 +14,7 @@ macro(spheral_add_executable) list(APPEND arg_DEPENDS_ON cuda) endif() - if (ENABLE_HIP) + if (SPHERAL_ENABLE_HIP) list(APPEND arg_DEPENDS_ON blt::hip) list(APPEND arg_DEPENDS_ON blt::hip_runtime) endif() diff --git a/cmake/spheral_cxx-config.cmake.in b/cmake/spheral_cxx-config.cmake.in index 0ebd6dec1f..29a1385495 100644 --- a/cmake/spheral_cxx-config.cmake.in +++ b/cmake/spheral_cxx-config.cmake.in @@ -10,7 +10,7 @@ if(NOT SPHERAL_FOUND) set(SPHERAL_ENABLE_MPI "@ENABLE_MPI@") set(SPHERAL_ENABLE_OPENMP "@ENABLE_OPENMP@") set(SPHERAL_ENABLE_CUDA "@ENABLE_CUDA@") - set(SPHERAL_ENABLE_HIP "@ENABLE_HIP@") + set(SPHERAL_ENABLE_HIP "@SPHERAL_ENABLE_HIP@") if(NOT axom_DIR) set(axom_DIR "@axom_DIR@" CACHE PATH "") endif() diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index a396ef2336..5e9dbb4991 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -183,7 +183,7 @@ def initconfig_hardware_entries(self): entries = super(Spheral, self).initconfig_hardware_entries() if spec.satisfies('+rocm'): - entries.append(cmake_cache_option("ENABLE_HIP", True)) + entries.append(cmake_cache_option("SPHERAL_ENABLE_HIP", True)) entries.append(cmake_cache_string("ROCM_PATH", spec["hip"].prefix)) if spec.satisfies('+cuda'): From 66c90f4c035c0eccc7eac477263c11b7a3bd1328 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 5 Sep 2025 09:48:04 -0700 Subject: [PATCH 25/52] Revert "Turn off mpi for axom when ~mpi" This reverts commit 63c17704270be751ecd0db34ec2cf082a342d14e. --- scripts/spack/packages/spheral/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 5e9dbb4991..6948789b42 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -69,7 +69,6 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('conduit@0.9.1 +shared +hdf5~hdf5_compat -test ~parmetis', type='build') depends_on('axom@0.9.0 +hdf5 -lua -examples -python -fortran', type='build') - depends_on('axom+mpi', when='+mpi') with when('+rocm') or when('+cuda'): depends_on('axom ~shared', type='build') From 21d150e0c55cfd90fbf6d86bec991a53d9fc0176 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 5 Sep 2025 09:48:06 -0700 Subject: [PATCH 26/52] Revert "Fix spack package typo" This reverts commit afd2f5801d0eac8acbbc96ec19b75704b5292c1e. --- scripts/spack/packages/spheral/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 6948789b42..2438cba837 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -167,7 +167,7 @@ def initconfig_compiler_entries(self): def initconfig_mpi_entries(self): spec = self.spec entries = [] - if spec.satisfies("+mpi"): + if spec.satisfied("+mpi"): entries = super(Spheral, self).initconfig_mpi_entries() # When on cray / flux systems we need to tell CMAKE the mpi flag explicitly if "cray-mpich" in spec: From 7797465b258f8810fe82b965b75bb4ef5258ae4d Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 5 Sep 2025 09:48:08 -0700 Subject: [PATCH 27/52] Revert "Removed umpire variant logic" This reverts commit a21597de08b5eac60f4bfe165fcbf6da319bcbc9. --- scripts/spack/packages/leos/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index ac6f343b7d..46af729b09 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -58,8 +58,8 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on("raja+rocm", when="+rocm") depends_on("umpire+cuda", when="+cuda") depends_on("umpire+rocm", when="+rocm") - depends_on("camp+cuda", when="+cuda") - depends_on("camp+rocm", when="+rocm") + depends_on("camp", when="+umpire ^umpire@2022:") + depends_on("camp+cuda", when="+cuda ^umpire@2022:") patch("patches/leos-8.5-umpire-import.patch", when="@8.5+rocm") From ff6f20305cb6b1d35b1a484a10277475c311d19b Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 5 Sep 2025 10:03:44 -0700 Subject: [PATCH 28/52] Reverted SPHERAL_ENABLE_HIP back to just ENABLE_HIP, added leos flags and libraries back, fixed prev bug added to spheral spack recipe --- cmake/SetupSpheral.cmake | 5 ++--- cmake/SpheralMacros.cmake | 2 +- cmake/spheral_cxx-config.cmake.in | 2 +- cmake/tpl/leos.cmake | 2 +- scripts/spack/packages/leos/package.py | 15 +++++++++++---- scripts/spack/packages/spheral/package.py | 4 ++-- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake index 714d5d524b..2bdb1c6696 100644 --- a/cmake/SetupSpheral.cmake +++ b/cmake/SetupSpheral.cmake @@ -69,7 +69,6 @@ option(SPHERAL_ENABLE_GSPH "Enable the GSPH package" ON) option(SPHERAL_ENABLE_SVPH "Enable the SVPH package" ON) option(SPHERAL_ENABLE_GLOBALDT_REDUCTION "Enable global allreduce for the time step" ON) option(SPHERAL_ENABLE_LONGCSDT "Enable longitudinal sound speed time step constraint" ON) -option(SPHERAL_ENABLE_HIP "Enable HIP" OFF) option(SPHERAL_ENABLE_LOGGER "Enable debug log printing" OFF) option(ENABLE_DEV_BUILD "Build separate internal C++ libraries for faster code development" OFF) @@ -98,10 +97,10 @@ if(ENABLE_CUDA) set(SPHERAL_ENABLE_CUDA ON) endif() -if(SPHERAL_ENABLE_HIP) - set(ENABLE_HIP ON) +if(ENABLE_HIP) list(APPEND SPHERAL_CXX_DEPENDS blt::hip) list(APPEND SPHERAL_CXX_DEPENDS blt::hip_runtime) + set(SPHERAL_ENABLE_HIP ON) endif() #-------------------------------------------------------------------------------# diff --git a/cmake/SpheralMacros.cmake b/cmake/SpheralMacros.cmake index 3de46f45c4..8438f20df9 100644 --- a/cmake/SpheralMacros.cmake +++ b/cmake/SpheralMacros.cmake @@ -14,7 +14,7 @@ macro(spheral_add_executable) list(APPEND arg_DEPENDS_ON cuda) endif() - if (SPHERAL_ENABLE_HIP) + if (ENABLE_HIP) list(APPEND arg_DEPENDS_ON blt::hip) list(APPEND arg_DEPENDS_ON blt::hip_runtime) endif() diff --git a/cmake/spheral_cxx-config.cmake.in b/cmake/spheral_cxx-config.cmake.in index 29a1385495..0ebd6dec1f 100644 --- a/cmake/spheral_cxx-config.cmake.in +++ b/cmake/spheral_cxx-config.cmake.in @@ -10,7 +10,7 @@ if(NOT SPHERAL_FOUND) set(SPHERAL_ENABLE_MPI "@ENABLE_MPI@") set(SPHERAL_ENABLE_OPENMP "@ENABLE_OPENMP@") set(SPHERAL_ENABLE_CUDA "@ENABLE_CUDA@") - set(SPHERAL_ENABLE_HIP "@SPHERAL_ENABLE_HIP@") + set(SPHERAL_ENABLE_HIP "@ENABLE_HIP@") if(NOT axom_DIR) set(axom_DIR "@axom_DIR@" CACHE PATH "") endif() diff --git a/cmake/tpl/leos.cmake b/cmake/tpl/leos.cmake index e1243466de..8d0d6527b9 100644 --- a/cmake/tpl/leos.cmake +++ b/cmake/tpl/leos.cmake @@ -1 +1 @@ -set(leos_libs libleos.a liblip-cpp.a) +set(leos_libs libleos.a libleos_C.a libyaml-cpp.a liblip-cpp.a ) diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index 46af729b09..0b70ea0e78 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -39,9 +39,10 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): version("8.3.1", sha256="35ae5a24185e29111886adaee66628f9e0b6ed3198e8c6381ef3c53bf662fd55") version("8.3.0", sha256="461fb0dc0672d5f284e392a8b70d9d50a035817aacb85a6843a5a75202a86cb5") - variant("mpi", default=True, description="Build wit MPI enabled") + variant("mpi", default=True, description="Build wit MPI enabled") variant("debug", default=False, description="Build debug code (-g -O0)") variant("filters", default=True , description="Build LEOS filter coding") + variant("yaml", default=True , description="Enable yaml features") variant("lto", default=False, description="Build w/-dlto when cuda-11") variant("cuda", default=False, description="Build LIP using RAJA + CUDA GPU code") variant("rocm", default=False, description="Build LIP using RAJA + ROCM GPU code") @@ -58,8 +59,8 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on("raja+rocm", when="+rocm") depends_on("umpire+cuda", when="+cuda") depends_on("umpire+rocm", when="+rocm") - depends_on("camp", when="+umpire ^umpire@2022:") depends_on("camp+cuda", when="+cuda ^umpire@2022:") + depends_on("camp+rocm", when="+rocm ^umpire@2022:") patch("patches/leos-8.5-umpire-import.patch", when="@8.5+rocm") @@ -97,15 +98,21 @@ def initconfig_package_entries(self): entries.append(cmake_cache_option("ENABLE_SILO", True)) entries.append(cmake_cache_path("SILO_PATH", spec["silo"].prefix)) entries.append(cmake_cache_option("ENABLE_PDB", True)) + entries.append(cmake_cache_option("ENABLE_FILTERS", "+filters" in spec)) + if spec.satisfies("+filters"): + entries.append(cmake_cache_option("USE_LIBXML2", False)) + entries.append(cmake_cache_option("USE_NOXML", True)) + entries.append(cmake_cache_option("ENABLE_YAML", "+yaml" in spec)) + entries.append(cmake_cache_option("ENABLE_MPI", '+mpi' in spec)) if spec.satisfies("+mpi"): - entries.append(cmake_cache_option("ENABLE_MPI", True)) entries.append(cmake_cache_path('-DMPI_C_COMPILER', spec['mpi'].mpicc)) entries.append(cmake_cache_path('-DMPI_CXX_COMPILER', spec['mpi'].mpicxx)) - features_disabled = ["PYTHON", "PYTHON_INTERFACE", "ZFP", "YAML", "TESTS", + features_disabled = ["PYTHON", "PYTHON_INTERFACE", "ZFP", "TESTS", "TOOLS", "EXAMPLES", "PARALLEL_EXAMPLES", "FORTRAN_INTERFACE"] for fd in features_disabled: entries.append(cmake_cache_option(f"ENABLE_{fd}", False)) entries.append(cmake_cache_option("ENABLE_CPP_LIP", True)) + entries.append(cmake_cache_option("ENABLE_C_LIP", True)) entries.append(cmake_cache_option("ENABLE_HDF", True)) entries.append(cmake_cache_path("HDF5_ROOT", spec["hdf5"].prefix)) entries.append(cmake_cache_path("EOS_DATA_ROOT_DIR", "/usr/gapps/data/eos")) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 2438cba837..e9f1de95bf 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -182,7 +182,7 @@ def initconfig_hardware_entries(self): entries = super(Spheral, self).initconfig_hardware_entries() if spec.satisfies('+rocm'): - entries.append(cmake_cache_option("SPHERAL_ENABLE_HIP", True)) + entries.append(cmake_cache_option("ENABLE_HIP", True)) entries.append(cmake_cache_string("ROCM_PATH", spec["hip"].prefix)) if spec.satisfies('+cuda'): @@ -255,8 +255,8 @@ def initconfig_package_entries(self): entries.append(cmake_cache_path('polytope_DIR', spec['polytope'].prefix)) + entries.append(cmake_cache_option('ENABLE_MPI', '+mpi' in spec)) if spec.satisfies("+mpi"): - entries.append(cmake_cache_option('ENABLE_MPI', True)) entries.append(cmake_cache_path('-DMPI_C_COMPILER', spec['mpi'].mpicc) ) entries.append(cmake_cache_path('-DMPI_CXX_COMPILER', spec['mpi'].mpicxx) ) From 295ff7ef286cf880346544e4f336cc88b1b3974c Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 5 Sep 2025 11:15:28 -0700 Subject: [PATCH 29/52] Forgot about typo in spack package --- scripts/spack/packages/spheral/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index e9f1de95bf..194e989528 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -167,7 +167,7 @@ def initconfig_compiler_entries(self): def initconfig_mpi_entries(self): spec = self.spec entries = [] - if spec.satisfied("+mpi"): + if spec.satisfies("+mpi"): entries = super(Spheral, self).initconfig_mpi_entries() # When on cray / flux systems we need to tell CMAKE the mpi flag explicitly if "cray-mpich" in spec: From 25a87363d42dacead8fc6dd18991229add4630a9 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 5 Sep 2025 11:28:52 -0700 Subject: [PATCH 30/52] libleos_C.a does not exist for some reason --- cmake/tpl/leos.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/tpl/leos.cmake b/cmake/tpl/leos.cmake index 8d0d6527b9..c6bc631891 100644 --- a/cmake/tpl/leos.cmake +++ b/cmake/tpl/leos.cmake @@ -1 +1 @@ -set(leos_libs libleos.a libleos_C.a libyaml-cpp.a liblip-cpp.a ) +set(leos_libs libleos.a libyaml-cpp.a liblip-cpp.a ) From dca07de0cb80431cb9a868b41188272473adea07 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Sat, 6 Sep 2025 09:38:42 -0700 Subject: [PATCH 31/52] Turn xml on for leos --- scripts/spack/packages/leos/package.py | 9 ++++----- scripts/spack/packages/spheral/package.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index 0b70ea0e78..52a669ff2d 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -39,10 +39,11 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): version("8.3.1", sha256="35ae5a24185e29111886adaee66628f9e0b6ed3198e8c6381ef3c53bf662fd55") version("8.3.0", sha256="461fb0dc0672d5f284e392a8b70d9d50a035817aacb85a6843a5a75202a86cb5") - variant("mpi", default=True, description="Build wit MPI enabled") + variant("mpi", default=True, description="Build wit MPI enabled") variant("debug", default=False, description="Build debug code (-g -O0)") - variant("filters", default=True , description="Build LEOS filter coding") + variant("filters", default=True, description="Build LEOS filter coding") variant("yaml", default=True , description="Enable yaml features") + variant("xml", default=True , description="Enable xml features") variant("lto", default=False, description="Build w/-dlto when cuda-11") variant("cuda", default=False, description="Build LIP using RAJA + CUDA GPU code") variant("rocm", default=False, description="Build LIP using RAJA + ROCM GPU code") @@ -99,9 +100,7 @@ def initconfig_package_entries(self): entries.append(cmake_cache_path("SILO_PATH", spec["silo"].prefix)) entries.append(cmake_cache_option("ENABLE_PDB", True)) entries.append(cmake_cache_option("ENABLE_FILTERS", "+filters" in spec)) - if spec.satisfies("+filters"): - entries.append(cmake_cache_option("USE_LIBXML2", False)) - entries.append(cmake_cache_option("USE_NOXML", True)) + entries.append(cmake_cache_option("ENABLE_XML", "+xml" in spec)) entries.append(cmake_cache_option("ENABLE_YAML", "+yaml" in spec)) entries.append(cmake_cache_option("ENABLE_MPI", '+mpi' in spec)) if spec.satisfies("+mpi"): diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 194e989528..4808616da9 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -87,7 +87,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') - depends_on('leos@8.5.0', type='build', when='+leos') + depends_on('leos@8.5.2+filters+yaml+xml+silo', type='build', when='+leos') # Forward MPI Variants mpi_tpl_list = ["hdf5", "conduit", "axom", "adiak~shared"] From 7b711b5c9c2e360c8bad7823d1b7b65519e4f8c1 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Sep 2025 09:20:50 -0700 Subject: [PATCH 32/52] Explicitly turn off xml for leos --- scripts/spack/packages/leos/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index 52a669ff2d..07c85c4cb0 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -42,8 +42,8 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): variant("mpi", default=True, description="Build wit MPI enabled") variant("debug", default=False, description="Build debug code (-g -O0)") variant("filters", default=True, description="Build LEOS filter coding") - variant("yaml", default=True , description="Enable yaml features") - variant("xml", default=True , description="Enable xml features") + variant("yaml", default=True, description="Enable yaml features") + variant("xml", default=False, description="Enable xml features") variant("lto", default=False, description="Build w/-dlto when cuda-11") variant("cuda", default=False, description="Build LIP using RAJA + CUDA GPU code") variant("rocm", default=False, description="Build LIP using RAJA + ROCM GPU code") From 0eeca882a3bc958e0e597dd3fe1d5bf1f44253cc Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Sep 2025 09:26:19 -0700 Subject: [PATCH 33/52] Do not use +xml variant in spheral --- scripts/spack/packages/spheral/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 4808616da9..3aa299bdef 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -87,7 +87,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') - depends_on('leos@8.5.2+filters+yaml+xml+silo', type='build', when='+leos') + depends_on('leos@8.5.2+filters+yaml~xml+silo', type='build', when='+leos') # Forward MPI Variants mpi_tpl_list = ["hdf5", "conduit", "axom", "adiak~shared"] From 84fd39a7ffff1120c66fa93d8ff3136fa9f0e09c Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Sep 2025 13:14:27 -0700 Subject: [PATCH 34/52] Switched to using find_package on hdf5 and LIP libraries, added logic to make some TPLs debug when build_type=Debug is added to the spack spec --- cmake/InstallTPLs.cmake | 46 ++++++++++++++++------- cmake/tpl/leos.cmake | 7 +++- scripts/spack/packages/leos/package.py | 1 - scripts/spack/packages/spheral/package.py | 9 +++++ 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index b15c1e6f3c..68942c25f4 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -49,7 +49,7 @@ if (NOT ENABLE_CXXONLY) list(APPEND BUILD_REQ_LIST ${SPHERAL_ROOT_DIR}/scripts/docs-requirements.txt) endif() - Spheral_Python_Env(python_build_env + Spheral_Python_Env(python_build_env REQUIREMENTS ${BUILD_REQ_LIST} PREFIX ${CMAKE_BINARY_DIR} ) @@ -74,7 +74,7 @@ if (NOT polyclipper_DIR) EXPORT spheral_cxx-targets DESTINATION lib/cmake) set_target_properties(PolyClipperAPI PROPERTIES EXPORT_NAME spheral::PolyClipperAPI) - message("Found PolyClipper External Package") + message("Found PolyClipper External Package.") else() list(APPEND SPHERAL_EXTERN_LIBS polyclipper) endif() @@ -111,7 +111,7 @@ if(adiak_FOUND) list(APPEND SPHERAL_BLT_DEPENDS adiak::adiak) list(APPEND SPHERAL_FP_TPLS adiak) list(APPEND SPHERAL_FP_DIRS ${adiak_DIR}) - message("Found Adiak External Package") + message("Found Adiak External Package.") endif() message("-----------------------------------------------------------------------------") @@ -132,7 +132,7 @@ if(POLYTOPE_FOUND) "${POLYTOPE_INSTALL_PREFIX}/${POLYTOPE_SITE_PACKAGES_PATH}/polytope.so not found") endif() endif() - message("Found Polytope External Package") + message("Found Polytope External Package.") else() list(APPEND SPHERAL_EXTERN_LIBS polytope) endif() @@ -150,20 +150,20 @@ if (SPHERAL_ENABLE_TIMERS) list(APPEND SPHERAL_BLT_DEPENDS caliper) list(APPEND SPHERAL_FP_TPLS caliper) list(APPEND SPHERAL_FP_DIRS ${caliper_DIR}) - message("Found Caliper External Package") + message("Found Caliper External Package.") endif() endif() message("-----------------------------------------------------------------------------") find_package(RAJA REQUIRED NO_DEFAULT_PATH PATHS ${raja_DIR}) -if (RAJA_FOUND) +if (RAJA_FOUND) message("Found RAJA External Package.") blt_convert_to_system_includes(TARGET RAJA) endif() message("-----------------------------------------------------------------------------") find_package(umpire REQUIRED NO_DEFAULT_PATH PATHS ${umpire_DIR}) -if (umpire_FOUND) +if (umpire_FOUND) message("Found umpire External Package.") blt_convert_to_system_includes(TARGET umpire) endif() @@ -176,9 +176,28 @@ if(chai_FOUND) blt_convert_to_system_includes(TARGET chai) endif() -list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire) -list(APPEND SPHERAL_FP_TPLS chai RAJA umpire) -list(APPEND SPHERAL_FP_DIRS ${chai_DIR} ${raja_DIR} ${umpire_DIR}) +# LIP +if(ENABLE_LEOS) + message("-----------------------------------------------------------------------------") + find_package(lip-cpp REQUIRED NO_DEFAULT_PATH PATHS ${leos_DIR}) + message("Found LIP External Package.") + blt_convert_to_system_includes(TARGET lip-cpp) + list(APPEND SPHERAL_BLT_DEPENDS lip-cpp) + list(APPEND SPHERAL_FP_TPLS lip-cpp) + list(APPEND SPHERAL_FP_DIRS ${leos_DIR}) +endif() + +message("-----------------------------------------------------------------------------") +# HDF5 +find_package(hdf5 REQUIRED NO_DEFAULT_PATH PATHS ${hdf5_DIR}) +if(hdf5_FOUND) + message("Found hdf5 External Package.") + blt_convert_to_system_includes(TARGET hdf5) +endif() + +list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire hdf5) +list(APPEND SPHERAL_FP_TPLS chai RAJA umpire hdf5) +list(APPEND SPHERAL_FP_DIRS ${chai_DIR} ${raja_DIR} ${umpire_DIR} ${hdf5_DIR}) set_property(GLOBAL PROPERTY SPHERAL_FP_TPLS ${SPHERAL_FP_TPLS}) set_property(GLOBAL PROPERTY SPHERAL_FP_DIRS ${SPHERAL_FP_DIRS}) @@ -187,19 +206,19 @@ message("----------------------------------------------------------------------- if (ENABLE_SUNDIALS) set(SUNDIALS_DIR "${sundials_DIR}") find_package(SUNDIALS REQUIRED NO_DEFAULT_PATH - COMPONENTS kinsol nvecparallel nvecmpiplusx nvecserial + COMPONENTS kinsol nvecparallel nvecmpiplusx nvecserial PATHS ${sundials_DIR}/lib64/cmake/sundials ${sundials_DIR}/lib/cmake/sundials) if(SUNDIALS_FOUND) list(APPEND SPHERAL_BLT_DEPENDS SUNDIALS::kinsol_static SUNDIALS::nvecparallel_static SUNDIALS::nvecmpiplusx_static SUNDIALS::nvecserial_static) list(APPEND SPHERAL_FP_TPLS SUNDIALS::kinsol_static SUNDIALS::nvecparallel_static SUNDIALS::nvecmpiplusx_static SUNDIALS::nvecserial_static) list(APPEND SPHERAL_FP_DIRS ${sundials_DIR}) - message("Found SUNDIALS External Package") + message("Found SUNDIALS External Package.") endif() endif() message("-----------------------------------------------------------------------------") # TPLs that must be imported -list(APPEND SPHERAL_EXTERN_LIBS boost eigen qhull silo hdf5) +list(APPEND SPHERAL_EXTERN_LIBS boost eigen qhull silo) blt_list_append( TO SPHERAL_EXTERN_LIBS ELEMENTS leos IF ENABLE_LEOS) blt_list_append( TO SPHERAL_EXTERN_LIBS ELEMENTS aneos IF ENABLE_ANEOS) @@ -231,4 +250,3 @@ if (NOT ENABLE_CXXONLY) DESTINATION ${CMAKE_INSTALL_PREFIX}/.venv/${SPHERAL_SITE_PACKAGES_PATH}/polytope/ ) endif() - diff --git a/cmake/tpl/leos.cmake b/cmake/tpl/leos.cmake index c6bc631891..a0bce668e2 100644 --- a/cmake/tpl/leos.cmake +++ b/cmake/tpl/leos.cmake @@ -1 +1,6 @@ -set(leos_libs libleos.a libyaml-cpp.a liblip-cpp.a ) +set(leos_libs libleos.a) +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + list(APPEND leos_libs libyaml-cppd.a) +else() + list(APPEND leos_libs libyaml-cpp.a) +endif() diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index 07c85c4cb0..a6f950bb41 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -40,7 +40,6 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): version("8.3.0", sha256="461fb0dc0672d5f284e392a8b70d9d50a035817aacb85a6843a5a75202a86cb5") variant("mpi", default=True, description="Build wit MPI enabled") - variant("debug", default=False, description="Build debug code (-g -O0)") variant("filters", default=True, description="Build LEOS filter coding") variant("yaml", default=True, description="Enable yaml features") variant("xml", default=False, description="Enable xml features") diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 3aa299bdef..9e2b634fd9 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -86,8 +86,10 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('polytope ~python', type='build', when="~python") depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') + depends_on('sundials build_type=Debug', when='+sundials build_type=Debug') depends_on('leos@8.5.2+filters+yaml~xml+silo', type='build', when='+leos') + depends_on('leos build_type=Debug', when='+leos build_type=Debug') # Forward MPI Variants mpi_tpl_list = ["hdf5", "conduit", "axom", "adiak~shared"] @@ -105,6 +107,11 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): for val in ROCmPackage.amdgpu_targets: depends_on(f"{ctpl} +rocm amdgpu_target={val}", type='build', when=f"+rocm amdgpu_target={val}") + # Forward debug variants + debug_tpl_list = gpu_tpl_list + ["hdf5", "adiak~shared"] + for ctpl in debug_tpl_list: + depends_on(f"{ctpl} build_type=Debug", when="build_type=Debug") + # ------------------------------------------------------------------------- # Conflicts # ------------------------------------------------------------------------- @@ -157,6 +164,8 @@ def cache_name(self): cache_spec += "+cuda" if spec.satisfies("+rocm"): cache_spec += "+rocm" + if spec.satisfies("build_type=Debug"): + cache_spec += "_debug" return f"{self._get_sys_type(spec)}-{cache_spec.replace(' ', '_')}.cmake" def initconfig_compiler_entries(self): From 5ae26420acdac4486f497b5eb7a13827be92d43a Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Sep 2025 13:50:37 -0700 Subject: [PATCH 35/52] Try turning yaml off for leos and changing name of LIP package in cmake logic --- cmake/InstallTPLs.cmake | 2 +- cmake/tpl/leos.cmake | 10 +++++----- scripts/spack/packages/leos/package.py | 2 +- scripts/spack/packages/spheral/package.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index 68942c25f4..879ade60ea 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -179,7 +179,7 @@ endif() # LIP if(ENABLE_LEOS) message("-----------------------------------------------------------------------------") - find_package(lip-cpp REQUIRED NO_DEFAULT_PATH PATHS ${leos_DIR}) + find_package(lip REQUIRED NO_DEFAULT_PATH PATHS ${leos_DIR}) message("Found LIP External Package.") blt_convert_to_system_includes(TARGET lip-cpp) list(APPEND SPHERAL_BLT_DEPENDS lip-cpp) diff --git a/cmake/tpl/leos.cmake b/cmake/tpl/leos.cmake index a0bce668e2..ea29086db0 100644 --- a/cmake/tpl/leos.cmake +++ b/cmake/tpl/leos.cmake @@ -1,6 +1,6 @@ set(leos_libs libleos.a) -if(CMAKE_BUILD_TYPE STREQUAL "Debug") - list(APPEND leos_libs libyaml-cppd.a) -else() - list(APPEND leos_libs libyaml-cpp.a) -endif() +# if(CMAKE_BUILD_TYPE STREQUAL "Debug") +# list(APPEND leos_libs libyaml-cppd.a) +# else() +# list(APPEND leos_libs libyaml-cpp.a) +# endif() diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index a6f950bb41..a35771795b 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -41,7 +41,7 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): variant("mpi", default=True, description="Build wit MPI enabled") variant("filters", default=True, description="Build LEOS filter coding") - variant("yaml", default=True, description="Enable yaml features") + variant("yaml", default=False, description="Enable yaml features") variant("xml", default=False, description="Enable xml features") variant("lto", default=False, description="Build w/-dlto when cuda-11") variant("cuda", default=False, description="Build LIP using RAJA + CUDA GPU code") diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 9e2b634fd9..300ed5c120 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -88,7 +88,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') depends_on('sundials build_type=Debug', when='+sundials build_type=Debug') - depends_on('leos@8.5.2+filters+yaml~xml+silo', type='build', when='+leos') + depends_on('leos@8.5.2+filters~yaml~xml+silo', type='build', when='+leos') depends_on('leos build_type=Debug', when='+leos build_type=Debug') # Forward MPI Variants From 38639073e61dfecd2687a570855ef57cb71f13c6 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Sep 2025 14:05:02 -0700 Subject: [PATCH 36/52] Change lip to LIP --- cmake/InstallTPLs.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index 879ade60ea..8b273c2796 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -179,7 +179,7 @@ endif() # LIP if(ENABLE_LEOS) message("-----------------------------------------------------------------------------") - find_package(lip REQUIRED NO_DEFAULT_PATH PATHS ${leos_DIR}) + find_package(LIP REQUIRED NO_DEFAULT_PATH PATHS ${leos_DIR}) message("Found LIP External Package.") blt_convert_to_system_includes(TARGET lip-cpp) list(APPEND SPHERAL_BLT_DEPENDS lip-cpp) From b771c6ed7f64d9baae665c9a5697a76381e4720d Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Sep 2025 14:18:46 -0700 Subject: [PATCH 37/52] LIP is incorrectly exported so reverting back to importing it the messy way --- cmake/InstallTPLs.cmake | 11 ----------- cmake/tpl/leos.cmake | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index 8b273c2796..6938918471 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -176,17 +176,6 @@ if(chai_FOUND) blt_convert_to_system_includes(TARGET chai) endif() -# LIP -if(ENABLE_LEOS) - message("-----------------------------------------------------------------------------") - find_package(LIP REQUIRED NO_DEFAULT_PATH PATHS ${leos_DIR}) - message("Found LIP External Package.") - blt_convert_to_system_includes(TARGET lip-cpp) - list(APPEND SPHERAL_BLT_DEPENDS lip-cpp) - list(APPEND SPHERAL_FP_TPLS lip-cpp) - list(APPEND SPHERAL_FP_DIRS ${leos_DIR}) -endif() - message("-----------------------------------------------------------------------------") # HDF5 find_package(hdf5 REQUIRED NO_DEFAULT_PATH PATHS ${hdf5_DIR}) diff --git a/cmake/tpl/leos.cmake b/cmake/tpl/leos.cmake index ea29086db0..3c2f1dd6dc 100644 --- a/cmake/tpl/leos.cmake +++ b/cmake/tpl/leos.cmake @@ -1,4 +1,4 @@ -set(leos_libs libleos.a) +set(leos_libs libleos.a liblip-cpp.a) # if(CMAKE_BUILD_TYPE STREQUAL "Debug") # list(APPEND leos_libs libyaml-cppd.a) # else() From 2edf1ab7f7bd4104a99d957a87d1b858dbf11c17 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Sep 2025 14:39:15 -0700 Subject: [PATCH 38/52] Fix hdf5 target names --- cmake/InstallTPLs.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index 6938918471..5c259d53a6 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -181,7 +181,7 @@ message("----------------------------------------------------------------------- find_package(hdf5 REQUIRED NO_DEFAULT_PATH PATHS ${hdf5_DIR}) if(hdf5_FOUND) message("Found hdf5 External Package.") - blt_convert_to_system_includes(TARGET hdf5) + blt_convert_to_system_includes(TARGET hdf5-shared hdf5-static) endif() list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire hdf5) From f082082a872b31731cafe24d193753c247b07850 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Sep 2025 16:22:34 -0700 Subject: [PATCH 39/52] Add correct hdf5 targets to lists --- cmake/InstallTPLs.cmake | 23 ++++++++++++++--------- cmake/tpl/hdf5.cmake | 3 --- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index 5c259d53a6..09317207a2 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -154,6 +154,17 @@ if (SPHERAL_ENABLE_TIMERS) endif() endif() +message("-----------------------------------------------------------------------------") +# HDF5 +find_package(hdf5 REQUIRED NO_DEFAULT_PATH PATHS ${hdf5_DIR}) +if(hdf5_FOUND) + message("Found HDF5 External Package.") + list(APPEND SPHERAL_BLT_DEPENDS hdf5-shared hdf5_hl-shared) + list(APPEND SPHERAL_FP_TPLS hdf5-shared hdf5_hl-shared) + list(APPEND SPHERAL_FP_DIRS ${hdf5_DIR}) + blt_convert_to_system_includes(TARGET hdf5-shared hdf5_hl-shared) +endif() + message("-----------------------------------------------------------------------------") find_package(RAJA REQUIRED NO_DEFAULT_PATH PATHS ${raja_DIR}) if (RAJA_FOUND) @@ -176,16 +187,10 @@ if(chai_FOUND) blt_convert_to_system_includes(TARGET chai) endif() -message("-----------------------------------------------------------------------------") -# HDF5 -find_package(hdf5 REQUIRED NO_DEFAULT_PATH PATHS ${hdf5_DIR}) -if(hdf5_FOUND) - message("Found hdf5 External Package.") - blt_convert_to_system_includes(TARGET hdf5-shared hdf5-static) -endif() -list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire hdf5) -list(APPEND SPHERAL_FP_TPLS chai RAJA umpire hdf5) + +list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire) +list(APPEND SPHERAL_FP_TPLS chai RAJA umpire) list(APPEND SPHERAL_FP_DIRS ${chai_DIR} ${raja_DIR} ${umpire_DIR} ${hdf5_DIR}) set_property(GLOBAL PROPERTY SPHERAL_FP_TPLS ${SPHERAL_FP_TPLS}) set_property(GLOBAL PROPERTY SPHERAL_FP_DIRS ${SPHERAL_FP_DIRS}) diff --git a/cmake/tpl/hdf5.cmake b/cmake/tpl/hdf5.cmake index 53341a8e5f..e825751e0c 100644 --- a/cmake/tpl/hdf5.cmake +++ b/cmake/tpl/hdf5.cmake @@ -1,5 +1,2 @@ set(${lib_name}_libs libhdf5.so libhdf5_hl.so) -if(ENABLE_STATIC_TPL) - string(REPLACE ".so" ".a;" ${lib_name}_libs ${${lib_name}_libs}) -endif() From 5ecc1ac84e9f769dda1ac4bc72d03a7c940ca7ad Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 8 Sep 2025 16:23:05 -0700 Subject: [PATCH 40/52] Remove duplicated hdf5_DIR in fp_dirs list --- cmake/InstallTPLs.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/InstallTPLs.cmake b/cmake/InstallTPLs.cmake index 09317207a2..387a4fdb78 100644 --- a/cmake/InstallTPLs.cmake +++ b/cmake/InstallTPLs.cmake @@ -191,7 +191,7 @@ endif() list(APPEND SPHERAL_BLT_DEPENDS chai camp RAJA umpire) list(APPEND SPHERAL_FP_TPLS chai RAJA umpire) -list(APPEND SPHERAL_FP_DIRS ${chai_DIR} ${raja_DIR} ${umpire_DIR} ${hdf5_DIR}) +list(APPEND SPHERAL_FP_DIRS ${chai_DIR} ${raja_DIR} ${umpire_DIR}) set_property(GLOBAL PROPERTY SPHERAL_FP_TPLS ${SPHERAL_FP_TPLS}) set_property(GLOBAL PROPERTY SPHERAL_FP_DIRS ${SPHERAL_FP_DIRS}) From d6ebe4febb00d3289c418b27f5237c96958a6504 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 9 Sep 2025 15:02:18 -0700 Subject: [PATCH 41/52] Build static cxx libraries if rocm is turned on, otherwise mysterious segfault occurs on exit --- scripts/spack/packages/spheral/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 300ed5c120..7fdc3ee40e 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -35,7 +35,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): variant('mpi', default=True, description='Enable MPI Support.') variant('openmp', default=True, description='Enable OpenMP Support.') variant('docs', default=False, description='Enable building Docs.') - variant('shared', default=True, description='Build C++ libs as shared.') + variant('shared', default=True, when="~rocm", description='Build C++ libs as shared.') variant('python', default=True, description='Build Python Dependencies.') variant('caliper', default=True, description='Enable Caliper timers.') variant('opensubdiv', default=True, description='Enable use of opensubdiv to do refinement.') @@ -116,6 +116,8 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): # Conflicts # ------------------------------------------------------------------------- conflicts("+cuda", when="+rocm") + conflicts("+shared", when="+rocm") + conflicts("~shared", when="+cuda") conflicts("%pgi") def _get_sys_type(self, spec): @@ -268,9 +270,7 @@ def initconfig_package_entries(self): if spec.satisfies("+mpi"): entries.append(cmake_cache_path('-DMPI_C_COMPILER', spec['mpi'].mpicc) ) entries.append(cmake_cache_path('-DMPI_CXX_COMPILER', spec['mpi'].mpicxx) ) - - if spec.satisfies("~shared") and spec.satisfies("~cuda"): - entries.append(cmake_cache_option('ENABLE_SHARED', False)) + entries.append(cmake_cache_option('ENABLE_SHARED', '+shared' in spec)) entries.append(cmake_cache_option('ENABLE_OPENMP', '+openmp' in spec)) entries.append(cmake_cache_option('ENABLE_DOCS', '+docs' in spec)) From 7ceae955a724591b417bb88771477539246688b2 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 10 Sep 2025 13:22:43 -0700 Subject: [PATCH 42/52] Changed umpire and raja versions based on feedback from Collette, reverted leos version back to 8.4.2, added memory type and exec space things to leos code for when we do end up using it, reverted back to using shared libraries for C++ libraries --- scripts/spack/packages/chai/package.py | 4 ++-- scripts/spack/packages/leos/package.py | 3 +-- scripts/spack/packages/raja/package.py | 2 +- scripts/spack/packages/spheral/package.py | 8 ++------ scripts/spack/packages/umpire/package.py | 2 +- src/LEOS/LEOS_bundle.cc | 13 +++++++++++-- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/scripts/spack/packages/chai/package.py b/scripts/spack/packages/chai/package.py index 7d7d6d7977..ee56e58d85 100644 --- a/scripts/spack/packages/chai/package.py +++ b/scripts/spack/packages/chai/package.py @@ -11,5 +11,5 @@ class Chai(BuiltinChai): version("develop", commit="b7babdc0c333baa68e53b026c63d65c48c8d8eb1", submodules=False) - depends_on("raja@2025.03.2", type="build", when="+raja") - depends_on("umpire@2025.03.0", type="build") + depends_on("raja@2025.03.0", type="build", when="+raja") + depends_on("umpire@2025.03.1", type="build") diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index a35771795b..f4f9d224b2 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -28,8 +28,7 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): version("8.5.2", sha256="0fd104fd8599c5349d5156a433df0aa04880c01eb0105c9318493fc17b3b5a6f", preferred=True) version("8.5.1", sha256="a072e48100bca21a594c6725158a0a7128f65ee4ce2aaa0be6e8fe55d3eff96a") version("8.5.0", sha256="49b6549ce5fbca8afdd58f2266591f6ce68341b2f37bf4302c08c217a353362a") - version("8.4.2.2", sha256="9ffdd60995fa2aecc408c71ec7bd0e52eae0803bc372ff43389bcb5a1e59bbe0") - version("8.4.2.1", sha256="589a2b664e1bc1d245816dd536c950065ec2a6dac3f16b9ed53fb86e4e79a4db") + #version("8.4.2.1", sha256="589a2b664e1bc1d245816dd536c950065ec2a6dac3f16b9ed53fb86e4e79a4db") version("8.4.2", sha256="08eb87580e30d7a1db72b1e1a457652dda9535df1c0caf7b5badb9cadf39f2a9") version("8.4.1", sha256="93abeea9e336e3a81cc6cc9de10b2a2fd61eda2a89abece50cac80fef58ec38b") version("8.4.0", sha256="233333d0ac1bd0fa3a4eb756248c6c996b98bccb8dd957d9fac9e744fb6ede6b") diff --git a/scripts/spack/packages/raja/package.py b/scripts/spack/packages/raja/package.py index d61e2b53f4..e89ff9a8e7 100644 --- a/scripts/spack/packages/raja/package.py +++ b/scripts/spack/packages/raja/package.py @@ -10,5 +10,5 @@ class Raja(BuiltinRaja): - version("2025.03.2", tag="v2025.03.2", submodules=False) + version("2025.03.0", tag="v2025.03.0", submodules=False) depends_on("camp@2025.03.0", type="build") diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 7fdc3ee40e..7438c0f47a 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -35,7 +35,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): variant('mpi', default=True, description='Enable MPI Support.') variant('openmp', default=True, description='Enable OpenMP Support.') variant('docs', default=False, description='Enable building Docs.') - variant('shared', default=True, when="~rocm", description='Build C++ libs as shared.') + variant('shared', default=True, description='Build C++ libs as shared.') variant('python', default=True, description='Build Python Dependencies.') variant('caliper', default=True, description='Enable Caliper timers.') variant('opensubdiv', default=True, description='Enable use of opensubdiv to do refinement.') @@ -88,7 +88,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') depends_on('sundials build_type=Debug', when='+sundials build_type=Debug') - depends_on('leos@8.5.2+filters~yaml~xml+silo', type='build', when='+leos') + depends_on('leos@8.4.2+filters~yaml~xml+silo', type='build', when='+leos') depends_on('leos build_type=Debug', when='+leos build_type=Debug') # Forward MPI Variants @@ -99,8 +99,6 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): # Forward CUDA/ROCM Variants gpu_tpl_list = ["raja", "umpire", "axom", "chai"] - if (LEOSpresent): - gpu_tpl_list.append("leos") for ctpl in gpu_tpl_list: for val in CudaPackage.cuda_arch_values: depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val}") @@ -116,8 +114,6 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): # Conflicts # ------------------------------------------------------------------------- conflicts("+cuda", when="+rocm") - conflicts("+shared", when="+rocm") - conflicts("~shared", when="+cuda") conflicts("%pgi") def _get_sys_type(self, spec): diff --git a/scripts/spack/packages/umpire/package.py b/scripts/spack/packages/umpire/package.py index e708c251c9..02c9c9edb3 100644 --- a/scripts/spack/packages/umpire/package.py +++ b/scripts/spack/packages/umpire/package.py @@ -10,5 +10,5 @@ class Umpire(BuiltinUmpire): - version("2025.03.0", tag="v2025.03.0", submodules=False) + version("2025.03.1", tag="v2025.03.1", submodules=False) depends_on("camp@2025.03.0", type="build") diff --git a/src/LEOS/LEOS_bundle.cc b/src/LEOS/LEOS_bundle.cc index 30918a8eff..d1e050ca1c 100644 --- a/src/LEOS/LEOS_bundle.cc +++ b/src/LEOS/LEOS_bundle.cc @@ -63,10 +63,13 @@ materialPtr(const eosnum_t eosnum, bundle.mMatPtrs[key] = matPtr; // Add the EOS functions we need - // LEOS::LEOS_FunctionOptions opts; + LEOS::LEOS_FunctionOptions opts; + opts.interpolantMemoryType(LEOS::MT_CPU); + opts.interpolantTemporaryMemoryType(LEOS::MT_CPU); + opts.execSpace(LEOS::ExecType_t::CPU); // opts.setTcalc(true); for (const auto& funcName: bundle.mFuncTemplate) { - auto funcPtr = matPtr->getFunction(funcName, LEOS::BIMOND); //, &opts); + auto funcPtr = matPtr->getFunction(funcName, LEOS::BIMOND, &opts); if (funcPtr->isValid()) bundle.mFuncPtrs[key][funcName] = funcPtr; } // CHECK(bundle.mFuncPtrs[key].size() == bundle.mFuncTemplate.size()); @@ -127,6 +130,11 @@ LEOS_bundle(): opts.units(LEOS::LEOS_UNITS_CGS); #ifdef USE_MPI opts.communicator(Communicator::communicator()); +#endif +#ifdef ENABLE_HIP + opts.permanentMemoryType(LEOS::MT_CPU); + opts.temporaryMemoryType(LEOS::MT_CPU); + opts.execSpace(LEOS::ExecType_t::CPU); #endif LEOS::startup(opts); } @@ -136,6 +144,7 @@ LEOS_bundle(): //------------------------------------------------------------------------------ LEOS_bundle:: ~LEOS_bundle() { + printf("Shutting down leos singleton\n"); LEOS::shutdown(); // Tell LEOS to close down and clean up } From dbfe84594ff8bdf7b7cf7d84349e6b5655e3516c Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 10 Sep 2025 13:37:03 -0700 Subject: [PATCH 43/52] Remove printf statement left in LEOS_bundle.cc --- RELEASE_NOTES.md | 3 +++ src/LEOS/LEOS_bundle.cc | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1f7132c128..40b47822a2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -36,6 +36,9 @@ Notable changes include: * Updated GitHub actions since GitLab mirror changed. * CHAI is no longer a submodule. * CHAI, RAJA, Umpire, and Camp are all brought in through Spack as external TPLs now. + * Umpire is updated to version 2025.03.1. + * RAJA is updated to version 2025.03.0. + * LEOs spack recipe is simplified. Version v2025.06.1 -- Release date 2025-07-21 ============================================== diff --git a/src/LEOS/LEOS_bundle.cc b/src/LEOS/LEOS_bundle.cc index d1e050ca1c..0b4d2f9147 100644 --- a/src/LEOS/LEOS_bundle.cc +++ b/src/LEOS/LEOS_bundle.cc @@ -144,7 +144,6 @@ LEOS_bundle(): //------------------------------------------------------------------------------ LEOS_bundle:: ~LEOS_bundle() { - printf("Shutting down leos singleton\n"); LEOS::shutdown(); // Tell LEOS to close down and clean up } From 2b9243e57e49c0e72304e82f92887a92d0327847 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 10 Sep 2025 13:42:18 -0700 Subject: [PATCH 44/52] One other addition to the release notes --- RELEASE_NOTES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 40b47822a2..da2b3dabc2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -39,6 +39,8 @@ Notable changes include: * Umpire is updated to version 2025.03.1. * RAJA is updated to version 2025.03.0. * LEOs spack recipe is simplified. + * Added spack logic to build TPLs that use CMake as debug when debug is turned on in the spec + by adding `build_type=Debug`. Version v2025.06.1 -- Release date 2025-07-21 ============================================== From 8ef5ced29092bc02dfa388ed795b299c58158298 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 11 Sep 2025 12:24:26 -0700 Subject: [PATCH 45/52] Bring yaml back to leos because it is apparently needed for reading ascii files --- cmake/tpl/leos.cmake | 2 +- scripts/spack/packages/leos/package.py | 3 +-- scripts/spack/packages/spheral/package.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmake/tpl/leos.cmake b/cmake/tpl/leos.cmake index 3c2f1dd6dc..5c11f313a6 100644 --- a/cmake/tpl/leos.cmake +++ b/cmake/tpl/leos.cmake @@ -1,4 +1,4 @@ -set(leos_libs libleos.a liblip-cpp.a) +set(leos_libs libleos.a liblip-cpp.a libyaml-cpp.a) # if(CMAKE_BUILD_TYPE STREQUAL "Debug") # list(APPEND leos_libs libyaml-cppd.a) # else() diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index f4f9d224b2..b2991e43c5 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -28,7 +28,6 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): version("8.5.2", sha256="0fd104fd8599c5349d5156a433df0aa04880c01eb0105c9318493fc17b3b5a6f", preferred=True) version("8.5.1", sha256="a072e48100bca21a594c6725158a0a7128f65ee4ce2aaa0be6e8fe55d3eff96a") version("8.5.0", sha256="49b6549ce5fbca8afdd58f2266591f6ce68341b2f37bf4302c08c217a353362a") - #version("8.4.2.1", sha256="589a2b664e1bc1d245816dd536c950065ec2a6dac3f16b9ed53fb86e4e79a4db") version("8.4.2", sha256="08eb87580e30d7a1db72b1e1a457652dda9535df1c0caf7b5badb9cadf39f2a9") version("8.4.1", sha256="93abeea9e336e3a81cc6cc9de10b2a2fd61eda2a89abece50cac80fef58ec38b") version("8.4.0", sha256="233333d0ac1bd0fa3a4eb756248c6c996b98bccb8dd957d9fac9e744fb6ede6b") @@ -40,7 +39,7 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): variant("mpi", default=True, description="Build wit MPI enabled") variant("filters", default=True, description="Build LEOS filter coding") - variant("yaml", default=False, description="Enable yaml features") + variant("yaml", default=True, description="Enable yaml features") variant("xml", default=False, description="Enable xml features") variant("lto", default=False, description="Build w/-dlto when cuda-11") variant("cuda", default=False, description="Build LIP using RAJA + CUDA GPU code") diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 7438c0f47a..102e7d8128 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -88,7 +88,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') depends_on('sundials build_type=Debug', when='+sundials build_type=Debug') - depends_on('leos@8.4.2+filters~yaml~xml+silo', type='build', when='+leos') + depends_on('leos@8.4.2+filters+yaml~xml+silo', type='build', when='+leos') depends_on('leos build_type=Debug', when='+leos build_type=Debug') # Forward MPI Variants From 08696b568308e70912d4b3d3f56bb33977b4c3ea Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 11 Sep 2025 14:43:00 -0700 Subject: [PATCH 46/52] Make preferred leos package the older one for now --- scripts/spack/packages/leos/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/spack/packages/leos/package.py b/scripts/spack/packages/leos/package.py index b2991e43c5..69eca1a47e 100644 --- a/scripts/spack/packages/leos/package.py +++ b/scripts/spack/packages/leos/package.py @@ -25,10 +25,10 @@ class Leos(CachedCMakePackage, CudaPackage, ROCmPackage): fileUrl = 'file://' + fileLoc url = os.path.join(fileUrl, "leos-8.4.1.tar.gz") - version("8.5.2", sha256="0fd104fd8599c5349d5156a433df0aa04880c01eb0105c9318493fc17b3b5a6f", preferred=True) + version("8.5.2", sha256="0fd104fd8599c5349d5156a433df0aa04880c01eb0105c9318493fc17b3b5a6f") version("8.5.1", sha256="a072e48100bca21a594c6725158a0a7128f65ee4ce2aaa0be6e8fe55d3eff96a") version("8.5.0", sha256="49b6549ce5fbca8afdd58f2266591f6ce68341b2f37bf4302c08c217a353362a") - version("8.4.2", sha256="08eb87580e30d7a1db72b1e1a457652dda9535df1c0caf7b5badb9cadf39f2a9") + version("8.4.2", sha256="08eb87580e30d7a1db72b1e1a457652dda9535df1c0caf7b5badb9cadf39f2a9", preferred=True) version("8.4.1", sha256="93abeea9e336e3a81cc6cc9de10b2a2fd61eda2a89abece50cac80fef58ec38b") version("8.4.0", sha256="233333d0ac1bd0fa3a4eb756248c6c996b98bccb8dd957d9fac9e744fb6ede6b") version("8.3.5", sha256="60d8298a5fc0dc056f33b39f664aab5ef75b4c4a4b3e1b80b22d769b39175db8") From 8b7786711085a764620ad07be0f50569c9115c2f Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 12 Sep 2025 14:10:33 -0700 Subject: [PATCH 47/52] Put link flags directly in blt call --- cmake/spheral/SpheralHandleTPL.cmake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmake/spheral/SpheralHandleTPL.cmake b/cmake/spheral/SpheralHandleTPL.cmake index 3ad16759ed..e2f6b9b91a 100644 --- a/cmake/spheral/SpheralHandleTPL.cmake +++ b/cmake/spheral/SpheralHandleTPL.cmake @@ -85,10 +85,8 @@ function(Spheral_Handle_TPL lib_name TPL_CMAKE_DIR) TREAT_INCLUDES_AS_SYSTEM ON INCLUDES ${${lib_name}_INCLUDE_DIR} LIBRARIES ${${lib_name}_LIBRARIES} + LINK_FLAGS ${${lib_name}_LINK_FLAGS} EXPORTABLE ON) - if(${lib_name}_LINK_FLAGS) - target_link_options(${lib_name} INTERFACE ${${lib_name}_LINK_FLAGS}) - endif() get_target_property(_is_imported ${lib_name} IMPORTED) if(NOT ${_is_imported}) install(TARGETS ${lib_name} From 0c5cb28ff00a8c93126379ad5e477cf376c47f41 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 15 Sep 2025 12:38:21 -0700 Subject: [PATCH 48/52] Add TPL dependencies and have leos get hip and cuda variants when necessary --- cmake/spheral/SpheralHandleTPL.cmake | 1 + scripts/spack/packages/spheral/package.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/spheral/SpheralHandleTPL.cmake b/cmake/spheral/SpheralHandleTPL.cmake index e2f6b9b91a..7113ae2f3a 100644 --- a/cmake/spheral/SpheralHandleTPL.cmake +++ b/cmake/spheral/SpheralHandleTPL.cmake @@ -85,6 +85,7 @@ function(Spheral_Handle_TPL lib_name TPL_CMAKE_DIR) TREAT_INCLUDES_AS_SYSTEM ON INCLUDES ${${lib_name}_INCLUDE_DIR} LIBRARIES ${${lib_name}_LIBRARIES} + DEPENDS_ON ${${lib_name}_DEPENDS} LINK_FLAGS ${${lib_name}_LINK_FLAGS} EXPORTABLE ON) get_target_property(_is_imported ${lib_name} IMPORTED) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 102e7d8128..1c6400dbf8 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -88,7 +88,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') depends_on('sundials build_type=Debug', when='+sundials build_type=Debug') - depends_on('leos@8.4.2+filters+yaml~xml+silo', type='build', when='+leos') + depends_on('leos@8.5.2+filters+yaml~xml+silo', type='build', when='+leos') depends_on('leos build_type=Debug', when='+leos build_type=Debug') # Forward MPI Variants @@ -99,6 +99,8 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): # Forward CUDA/ROCM Variants gpu_tpl_list = ["raja", "umpire", "axom", "chai"] + if LEOSpresent: + gpu_tpl_list.append("leos") for ctpl in gpu_tpl_list: for val in CudaPackage.cuda_arch_values: depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val}") From 444e36b7fb62bdb435e188a153128b616766e63e Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 15 Sep 2025 14:37:33 -0700 Subject: [PATCH 49/52] Change leos version back to 8.5.0 and ensure it is only a dependency when +leos --- scripts/spack/packages/spheral/package.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 1c6400dbf8..5e59dad809 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -88,7 +88,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') depends_on('sundials build_type=Debug', when='+sundials build_type=Debug') - depends_on('leos@8.5.2+filters+yaml~xml+silo', type='build', when='+leos') + depends_on('leos@8.5.0+filters+yaml~xml+silo', type='build', when='+leos') depends_on('leos build_type=Debug', when='+leos build_type=Debug') # Forward MPI Variants @@ -98,14 +98,16 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on(f"{ctpl} {mpiv}", type='build', when=f"{mpiv}") # Forward CUDA/ROCM Variants - gpu_tpl_list = ["raja", "umpire", "axom", "chai"] - if LEOSpresent: - gpu_tpl_list.append("leos") - for ctpl in gpu_tpl_list: + def set_gpu_variants(ctpl, cond=""): for val in CudaPackage.cuda_arch_values: - depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val}") + depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val} {cond}") for val in ROCmPackage.amdgpu_targets: - depends_on(f"{ctpl} +rocm amdgpu_target={val}", type='build', when=f"+rocm amdgpu_target={val}") + depends_on(f"{ctpl} +rocm amdgpu_target={val}", type='build', when=f"+rocm amdgpu_target={val} {cond}") + gpu_tpl_list = ["raja", "umpire", "axom", "chai"] + for ctpl in gpu_tpl_list: + set_gpu_variants(ctpl) + if LEOSpresent: + set_gpu_variants("leos", "+leos") # Forward debug variants debug_tpl_list = gpu_tpl_list + ["hdf5", "adiak~shared"] From 009dc7259ac87caadbd7cb40feb19a2a1f7173b2 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 15 Sep 2025 15:37:42 -0700 Subject: [PATCH 50/52] It seems leos 8.5.2 is more stable than 8.5.0 even if something is broken --- scripts/spack/packages/spheral/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index 5e59dad809..ac035890e4 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -88,7 +88,7 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') depends_on('sundials build_type=Debug', when='+sundials build_type=Debug') - depends_on('leos@8.5.0+filters+yaml~xml+silo', type='build', when='+leos') + depends_on('leos@8.5.2+filters+yaml~xml+silo', type='build', when='+leos') depends_on('leos build_type=Debug', when='+leos build_type=Debug') # Forward MPI Variants From 63ffbbdc563561374bd51b8567c4cfcd79657e8b Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 16 Sep 2025 10:58:54 -0700 Subject: [PATCH 51/52] Reverting back to leos 8.4.2 without rocm versions but added note for when this is revisited --- scripts/spack/packages/spheral/package.py | 24 ++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scripts/spack/packages/spheral/package.py b/scripts/spack/packages/spheral/package.py index ac035890e4..3fcd8dcda8 100644 --- a/scripts/spack/packages/spheral/package.py +++ b/scripts/spack/packages/spheral/package.py @@ -88,8 +88,12 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on('sundials@7.0.0 ~shared cxxstd=17 cppflags="-fPIC"', type='build', when='+sundials') depends_on('sundials build_type=Debug', when='+sundials build_type=Debug') - depends_on('leos@8.5.2+filters+yaml~xml+silo', type='build', when='+leos') - depends_on('leos build_type=Debug', when='+leos build_type=Debug') + # Forward CUDA/ROCM Variants + def set_gpu_variants(ctpl, cond=""): + for val in CudaPackage.cuda_arch_values: + depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val} {cond}") + for val in ROCmPackage.amdgpu_targets: + depends_on(f"{ctpl} +rocm amdgpu_target={val}", type='build', when=f"+rocm amdgpu_target={val} {cond}") # Forward MPI Variants mpi_tpl_list = ["hdf5", "conduit", "axom", "adiak~shared"] @@ -97,23 +101,21 @@ class Spheral(CachedCMakePackage, CudaPackage, ROCmPackage): for mpiv in ["+mpi", "~mpi"]: depends_on(f"{ctpl} {mpiv}", type='build', when=f"{mpiv}") - # Forward CUDA/ROCM Variants - def set_gpu_variants(ctpl, cond=""): - for val in CudaPackage.cuda_arch_values: - depends_on(f"{ctpl} +cuda cuda_arch={val}", type='build', when=f"+cuda cuda_arch={val} {cond}") - for val in ROCmPackage.amdgpu_targets: - depends_on(f"{ctpl} +rocm amdgpu_target={val}", type='build', when=f"+rocm amdgpu_target={val} {cond}") gpu_tpl_list = ["raja", "umpire", "axom", "chai"] for ctpl in gpu_tpl_list: set_gpu_variants(ctpl) - if LEOSpresent: - set_gpu_variants("leos", "+leos") - # Forward debug variants + # Forward debug variants debug_tpl_list = gpu_tpl_list + ["hdf5", "adiak~shared"] for ctpl in debug_tpl_list: depends_on(f"{ctpl} build_type=Debug", when="build_type=Debug") + depends_on('leos@8.4.2+filters+yaml~xml+silo', type='build', when='+leos') + depends_on('leos build_type=Debug', when='+leos build_type=Debug') + # TODO: Get leos working with +rocm variant using 8.5.2 + # if LEOSpresent: + # set_gpu_variants("leos", "+leos") + # ------------------------------------------------------------------------- # Conflicts # ------------------------------------------------------------------------- From a0799a01fc9d689305765108104b58dde47beb60 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 16 Sep 2025 12:46:28 -0700 Subject: [PATCH 52/52] Fixed copy and assignment constructors for QI and CHI to avoid copying MA, changed view call to be a static_cast --- src/Utilities/CubicHermiteInterpolator.cc | 4 ++-- src/Utilities/CubicHermiteInterpolator.hh | 14 +------------- src/Utilities/QuadraticInterpolator.cc | 4 ++-- src/Utilities/QuadraticInterpolator.hh | 14 +------------- 4 files changed, 6 insertions(+), 30 deletions(-) diff --git a/src/Utilities/CubicHermiteInterpolator.cc b/src/Utilities/CubicHermiteInterpolator.cc index 72155034bc..19221c0e40 100644 --- a/src/Utilities/CubicHermiteInterpolator.cc +++ b/src/Utilities/CubicHermiteInterpolator.cc @@ -20,7 +20,7 @@ namespace Spheral { // Copy constructor //------------------------------------------------------------------------------ CubicHermiteInterpolator::CubicHermiteInterpolator(const CubicHermiteInterpolator& rhs) : - CHIView(rhs) { + CHIView() { mVec = rhs.mVec; initializeMA(); } @@ -31,7 +31,6 @@ CubicHermiteInterpolator::CubicHermiteInterpolator(const CubicHermiteInterpolato CubicHermiteInterpolator& CubicHermiteInterpolator::operator=(const CubicHermiteInterpolator& rhs) { if (this != &rhs) { - CHIView::operator=(rhs); mVec = rhs.mVec; initializeMA(); } @@ -84,6 +83,7 @@ CubicHermiteInterpolator::~CubicHermiteInterpolator() { void CubicHermiteInterpolator::initializeMA() { + mVals.free(); mVals = chai::makeManagedArray(mVec.data(), mVec.size(), chai::CPU, false); } diff --git a/src/Utilities/CubicHermiteInterpolator.hh b/src/Utilities/CubicHermiteInterpolator.hh index dc5dd3b5fe..a3fbabba57 100644 --- a/src/Utilities/CubicHermiteInterpolator.hh +++ b/src/Utilities/CubicHermiteInterpolator.hh @@ -53,16 +53,6 @@ public: void move(chai::ExecutionSpace space) { mVals.move(space); } SPHERAL_HOST_DEVICE double* data() const { return mVals.data(); } - SPHERAL_HOST CHIView(size_t N, - double xmin, - double xmax, - double xstep, - ContainerType const& vals) : - mN(N), - mXmin(xmin), - mXmax(xmax), - mXstep(xstep), - mVals(vals) { mVals.registerTouch(chai::CPU); } protected: //--------------------------- Protected Interface --------------------------// // Member data @@ -115,9 +105,7 @@ public: // Force interpolation to be monotonic (may introduce structure between tabulated points) void makeMonotonic(); - CHIView view() { - return CHIView(mN, mXmin, mXmax, mXstep, mVals); - } + CHIView view() { return static_cast(*this); } private: //--------------------------- Private Interface --------------------------// diff --git a/src/Utilities/QuadraticInterpolator.cc b/src/Utilities/QuadraticInterpolator.cc index 104689716c..2d09cebd9d 100644 --- a/src/Utilities/QuadraticInterpolator.cc +++ b/src/Utilities/QuadraticInterpolator.cc @@ -18,7 +18,7 @@ namespace Spheral { //------------------------------------------------------------------------------ QuadraticInterpolator::QuadraticInterpolator(const QuadraticInterpolator& rhs) : - QIView(rhs) { + QIView() { mVec = rhs.mVec; initializeMA(); } @@ -29,7 +29,6 @@ QuadraticInterpolator::QuadraticInterpolator(const QuadraticInterpolator& rhs) QuadraticInterpolator& QuadraticInterpolator::operator=(const QuadraticInterpolator& rhs) { if (this != &rhs) { - QIView::operator=(rhs); mVec = rhs.mVec; initializeMA(); } @@ -93,6 +92,7 @@ QuadraticInterpolator::initialize(double xmin, void QuadraticInterpolator::initializeMA() { + mcoeffs.free(); mcoeffs = chai::makeManagedArray(mVec.data(), mVec.size(), chai::CPU, false); } diff --git a/src/Utilities/QuadraticInterpolator.hh b/src/Utilities/QuadraticInterpolator.hh index 1569fdfcfa..746205fb28 100644 --- a/src/Utilities/QuadraticInterpolator.hh +++ b/src/Utilities/QuadraticInterpolator.hh @@ -50,16 +50,6 @@ public: void move(chai::ExecutionSpace space) { mcoeffs.move(space); } SPHERAL_HOST_DEVICE double* data() const { return mcoeffs.data(); } - SPHERAL_HOST QIView(size_t N1, - double xmin, - double xmax, - double xstep, - ContainerType const& vals) : - mN1(N1), - mXmin(xmin), - mXmax(xmax), - mXstep(xstep), - mcoeffs(vals) { mcoeffs.registerTouch(chai::CPU); } protected: //--------------------------- Private Interface --------------------------// // Member data @@ -85,9 +75,7 @@ public: void initialize(double xmin, double xmax, size_t n, const Func& f); void initialize(double xmin, double xmax, const std::vector& yvals); - QIView view() { - return QIView(mN1, mXmin, mXmax, mXstep, mcoeffs); - } + QIView view() { return static_cast(*this); } private: std::vector mVec; void initializeMA();