Skip to content

Commit 88f9e68

Browse files
TomMeltTomMelt
committed
feat: added metadata info on neighbours
add metadata info for periodic and non-periodic neighbours i.e., halo sizes, starting indices etc. add an associated unit test to check values are read correctly from file.
1 parent 502e126 commit 88f9e68

File tree

6 files changed

+524
-7
lines changed

6 files changed

+524
-7
lines changed

‎core/src/ModelMetadata.cpp‎

Lines changed: 81 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
#include "include/IStructure.hpp"
1111
#include "include/NextsimModule.hpp"
1212
#include "include/gridNames.hpp"
13+
#include "mpi.h"
14+
#include <cstddef>
15+
#include <vector>
1316

1417
#ifdef USE_MPI
1518
#include <ncDim.h>
@@ -39,6 +42,75 @@ void ModelMetadata::setMpiMetadata(MPI_Comm comm)
3942
MPI_Comm_rank(mpiComm, &mpiMyRank);
4043
}
4144

45+
void ModelMetadata::readNeighbourData(netCDF::NcFile& ncFile)
46+
{
47+
netCDF::NcGroup neighbourGroup(ncFile.getGroup(neighbourName));
48+
std::string varName {};
49+
for (auto edge : edges) {
50+
size_t nStart {}; // start point in metadata arrays
51+
size_t count {}; // number of elements to read from metadata arrays
52+
std::vector<int> numNeighbours = std::vector<int>(mpiSize, 0);
53+
std::vector<int> offsets = std::vector<int>(mpiSize, 0);
54+
55+
// non-periodic neighbours
56+
varName = edgeNames[edge] + "_neighbours";
57+
neighbourGroup.getVar(varName).getVar(
58+
{ 0 }, { static_cast<size_t>(mpiSize) }, &numNeighbours[0]);
59+
60+
// compute start index for each process
61+
MPI_Exscan(&numNeighbours[mpiMyRank], &nStart, 1, MPI_INT, MPI_SUM, mpiComm);
62+
// how many elements to read for each process
63+
count = numNeighbours[mpiMyRank];
64+
65+
if (count) {
66+
// initialize neighbour info to zero and correct size
67+
neighbourRanks[edge].resize(count, 0);
68+
neighbourExtents[edge].resize(count, 0);
69+
neighbourHaloStarts[edge].resize(count, 0);
70+
71+
varName = edgeNames[edge] + "_neighbour_ids";
72+
neighbourGroup.getVar(varName).getVar({ nStart }, { count }, &neighbourRanks[edge][0]);
73+
74+
varName = edgeNames[edge] + "_neighbour_halos";
75+
neighbourGroup.getVar(varName).getVar(
76+
{ nStart }, { count }, &neighbourExtents[edge][0]);
77+
78+
varName = edgeNames[edge] + "_neighbour_halo_starts";
79+
neighbourGroup.getVar(varName).getVar(
80+
{ nStart }, { count }, &neighbourHaloStarts[edge][0]);
81+
}
82+
83+
// periodic neighbours
84+
varName = edgeNames[edge] + "_neighbours_periodic";
85+
neighbourGroup.getVar(varName).getVar(
86+
{ 0 }, { static_cast<size_t>(mpiSize) }, &numNeighbours[0]);
87+
88+
// compute start index for each process
89+
MPI_Exscan(&numNeighbours[mpiMyRank], &nStart, 1, MPI_INT, MPI_SUM, mpiComm);
90+
// how many elements to read for each process
91+
count = numNeighbours[mpiMyRank];
92+
93+
if (count) {
94+
// initialize neighbour info to zero and correct size
95+
neighbourRanksPeriodic[edge].resize(count, 0);
96+
neighbourExtentsPeriodic[edge].resize(count, 0);
97+
neighbourHaloStartsPeriodic[edge].resize(count, 0);
98+
99+
varName = edgeNames[edge] + "_neighbour_ids_periodic";
100+
neighbourGroup.getVar(varName).getVar(
101+
{ nStart }, { count }, &neighbourRanksPeriodic[edge][0]);
102+
103+
varName = edgeNames[edge] + "_neighbour_halos_periodic";
104+
neighbourGroup.getVar(varName).getVar(
105+
{ nStart }, { count }, &neighbourExtentsPeriodic[edge][0]);
106+
107+
varName = edgeNames[edge] + "_neighbour_halo_starts_periodic";
108+
neighbourGroup.getVar(varName).getVar(
109+
{ nStart }, { count }, &neighbourHaloStartsPeriodic[edge][0]);
110+
}
111+
}
112+
}
113+
42114
void ModelMetadata::getPartitionMetadata(std::string partitionFile)
43115
{
44116
// TODO: Move the reading of the partition file to its own class
@@ -53,11 +125,15 @@ void ModelMetadata::getPartitionMetadata(std::string partitionFile)
53125
globalExtentX = ncFile.getDim("NX").getSize();
54126
globalExtentY = ncFile.getDim("NY").getSize();
55127
netCDF::NcGroup bboxGroup(ncFile.getGroup(bboxName));
56-
std::vector<size_t> index(1, mpiMyRank);
57-
bboxGroup.getVar("domain_x").getVar(index, &localCornerX);
58-
bboxGroup.getVar("domain_y").getVar(index, &localCornerY);
59-
bboxGroup.getVar("domain_extent_x").getVar(index, &localExtentX);
60-
bboxGroup.getVar("domain_extent_y").getVar(index, &localExtentY);
128+
129+
std::vector<size_t> rank(1, mpiMyRank);
130+
bboxGroup.getVar("domain_x").getVar(rank, &localCornerX);
131+
bboxGroup.getVar("domain_y").getVar(rank, &localCornerY);
132+
bboxGroup.getVar("domain_extent_x").getVar(rank, &localExtentX);
133+
bboxGroup.getVar("domain_extent_y").getVar(rank, &localExtentY);
134+
135+
readNeighbourData(ncFile);
136+
61137
ncFile.close();
62138
}
63139

‎core/src/include/ModelMetadata.hpp‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@
1313
#include "include/ModelState.hpp"
1414
#include "include/Time.hpp"
1515

16+
#include <ncFile.h>
1617
#include <string>
18+
#include <vector>
1719

1820
#ifdef USE_MPI
1921
#include <mpi.h>
@@ -93,13 +95,34 @@ class ModelMetadata {
9395
*/
9496
void getPartitionMetadata(std::string partitionFile);
9597

98+
enum Edge { BOTTOM, RIGHT, TOP, LEFT, N_EDGE };
99+
// An array to allow the edges to be accessed in the correct order.
100+
static constexpr std::array<Edge, N_EDGE> edges = { BOTTOM, RIGHT, TOP, LEFT };
101+
std::array<std::string, N_EDGE> edgeNames = { "bottom", "right", "top", "left" };
102+
96103
MPI_Comm mpiComm;
97104
int mpiSize = 0;
98105
int mpiMyRank = -1;
99-
int localCornerX, localCornerY, localExtentX, localExtentY, globalExtentX, globalExtentY;
106+
int localCornerX, localCornerY;
107+
int localExtentX, localExtentY;
108+
int globalExtentX, globalExtentY;
109+
// mpi rank ID and extent for each edge direction
110+
std::array<std::vector<int>, N_EDGE> neighbourRanks;
111+
std::array<std::vector<int>, N_EDGE> neighbourExtents;
112+
std::array<std::vector<int>, N_EDGE> neighbourHaloStarts;
113+
std::array<std::vector<int>, N_EDGE> neighbourRanksPeriodic;
114+
std::array<std::vector<int>, N_EDGE> neighbourExtentsPeriodic;
115+
std::array<std::vector<int>, N_EDGE> neighbourHaloStartsPeriodic;
100116
#endif
101117

102118
private:
119+
/*!
120+
* @brief Read neighbour metadata from netCDF file
121+
*
122+
* @param netCDF file with partition metadata
123+
*/
124+
void readNeighbourData(netCDF::NcFile& ncFile);
125+
103126
TimePoint m_time;
104127
ConfigMap m_config;
105128

@@ -116,6 +139,7 @@ class ModelMetadata {
116139
bool hasParameters;
117140
#ifdef USE_MPI
118141
const std::string bboxName = "bounding_boxes";
142+
const std::string neighbourName = "connectivity";
119143
#endif
120144
};
121145

‎core/test/CMakeLists.txt‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,24 @@ if(ENABLE_MPI)
3232
ncgen -b -o partition_metadata_2.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.cdl
3333
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_2.cdl
3434
)
35+
add_custom_command(
36+
OUTPUT partition_metadata_3_cb.nc partition_metadata_3_pb.nc
37+
COMMAND
38+
ncgen -b -o partition_metadata_3_cb.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3_cb.cdl
39+
COMMAND
40+
ncgen -b -o partition_metadata_3_pb.nc ${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3_pb.cdl
41+
DEPENDS
42+
${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3_cb.cdl
43+
${CMAKE_CURRENT_SOURCE_DIR}/partition_metadata_3_pb.cdl
44+
)
3545
add_custom_target(
3646
generate_partition_files
3747
ALL
38-
DEPENDS partition_metadata_3.nc partition_metadata_2.nc
48+
DEPENDS
49+
partition_metadata_3.nc
50+
partition_metadata_2.nc
51+
partition_metadata_3_cb.nc
52+
partition_metadata_3_pb.nc
3953
)
4054

4155
add_executable(testRectGrid_MPI3 "RectGrid_test.cpp" "MainMPI.cpp")
@@ -49,6 +63,20 @@ if(ENABLE_MPI)
4963
)
5064
target_link_libraries(testRectGrid_MPI3 PRIVATE nextsimlib doctest::doctest)
5165

66+
add_executable(testModelMetadata_MPI3 "ModelMetadata_test.cpp" "MainMPI.cpp")
67+
target_compile_definitions(
68+
testModelMetadata_MPI3
69+
PRIVATE
70+
USE_MPI
71+
TEST_FILES_DIR=\"${CMAKE_CURRENT_BINARY_DIR}\"
72+
TEST_FILE_SOURCE=\"${CMAKE_CURRENT_SOURCE_DIR}\"
73+
)
74+
target_include_directories(
75+
testModelMetadata_MPI3
76+
PRIVATE ${MODEL_INCLUDE_DIR} "${ModulesRoot}/StructureModule"
77+
)
78+
target_link_libraries(testModelMetadata_MPI3 PRIVATE nextsimlib doctest::doctest)
79+
5280
add_executable(testParaGrid_MPI2 "ParaGrid_test.cpp" "MainMPI.cpp")
5381
target_compile_definitions(
5482
testParaGrid_MPI2

‎core/test/ModelMetadata_test.cpp‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/*!
2+
* @file ModelMetadata_test.cpp
3+
*
4+
* @date 18 Nov 2024
5+
* @author Tom Meltzer <tdm39@cam.ac.uk>
6+
*/
7+
8+
#include <doctest/extensions/doctest_mpi.h>
9+
#include <iostream>
10+
11+
#include "ModelMetadata.hpp"
12+
13+
const std::string testFilesDir = TEST_FILES_DIR;
14+
const std::string filename = testFilesDir + "/paraGrid_test.nc";
15+
const std::string diagFile = "paraGrid_diag.nc";
16+
const std::string dateString = "2000-01-01T00:00:00Z";
17+
const std::string partitionFilenameCB = testFilesDir + "/partition_metadata_3_cb.nc";
18+
const std::string partitionFilenamePB = testFilesDir + "/partition_metadata_3_pb.nc";
19+
20+
namespace Nextsim {
21+
22+
TEST_SUITE_BEGIN("ModelMetadata");
23+
MPI_TEST_CASE("Test getPartitionMetadata closed boundary", 3)
24+
{
25+
ModelMetadata metadata(partitionFilenameCB, test_comm);
26+
REQUIRE(metadata.mpiComm == test_comm);
27+
// this metadata is specific to the non-periodic boundary conditions
28+
if (test_rank == 0) {
29+
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT].size() == 0);
30+
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT] == std::vector<int> { 2 });
31+
REQUIRE(metadata.neighbourExtents[ModelMetadata::RIGHT] == std::vector<int> { 4 });
32+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::RIGHT] == std::vector<int> { 0 });
33+
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM].size() == 0);
34+
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP] == std::vector<int> { 1 });
35+
REQUIRE(metadata.neighbourExtents[ModelMetadata::TOP] == std::vector<int> { 7 });
36+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::TOP] == std::vector<int> { 0 });
37+
} else if (test_rank == 1) {
38+
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT].size() == 0);
39+
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT] == std::vector<int> { 2 });
40+
REQUIRE(metadata.neighbourExtents[ModelMetadata::RIGHT] == std::vector<int> { 5 });
41+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::RIGHT] == std::vector<int> { 12 });
42+
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM] == std::vector<int> { 0 });
43+
REQUIRE(metadata.neighbourExtents[ModelMetadata::BOTTOM] == std::vector<int> { 7 });
44+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::BOTTOM] == std::vector<int> { 21 });
45+
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP].size() == 0);
46+
} else if (test_rank == 2) {
47+
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT] == std::vector<int> { 0, 1 });
48+
REQUIRE(metadata.neighbourExtents[ModelMetadata::LEFT] == std::vector<int> { 4, 5 });
49+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::LEFT] == std::vector<int> { 6, 6 });
50+
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT].size() == 0);
51+
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM].size() == 0);
52+
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP].size() == 0);
53+
} else {
54+
std::cerr << "only valid for 3 ranks" << std::endl;
55+
exit(1);
56+
}
57+
58+
// This metadata is specific to the periodic boundary conditions.
59+
// They are all zero because the input metadata file `partitionFilenameCB` does not use periodic
60+
// boundary conditions.
61+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::LEFT].size() == 0);
62+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::RIGHT].size() == 0);
63+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::BOTTOM].size() == 0);
64+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::TOP].size() == 0);
65+
}
66+
67+
MPI_TEST_CASE("Test getPartitionMetadata periodic boundary", 3)
68+
{
69+
ModelMetadata metadata(partitionFilenamePB, test_comm);
70+
REQUIRE(metadata.mpiComm == test_comm);
71+
// this metadata should be identical to the Closed Boundary version so we check it again
72+
if (test_rank == 0) {
73+
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT].size() == 0);
74+
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT] == std::vector<int> { 2 });
75+
REQUIRE(metadata.neighbourExtents[ModelMetadata::RIGHT] == std::vector<int> { 4 });
76+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::RIGHT] == std::vector<int> { 0 });
77+
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM].size() == 0);
78+
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP] == std::vector<int> { 1 });
79+
REQUIRE(metadata.neighbourExtents[ModelMetadata::TOP] == std::vector<int> { 7 });
80+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::TOP] == std::vector<int> { 0 });
81+
} else if (test_rank == 1) {
82+
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT].size() == 0);
83+
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT] == std::vector<int> { 2 });
84+
REQUIRE(metadata.neighbourExtents[ModelMetadata::RIGHT] == std::vector<int> { 5 });
85+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::RIGHT] == std::vector<int> { 12 });
86+
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM] == std::vector<int> { 0 });
87+
REQUIRE(metadata.neighbourExtents[ModelMetadata::BOTTOM] == std::vector<int> { 7 });
88+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::BOTTOM] == std::vector<int> { 21 });
89+
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP].size() == 0);
90+
} else if (test_rank == 2) {
91+
REQUIRE(metadata.neighbourRanks[ModelMetadata::LEFT] == std::vector<int> { 0, 1 });
92+
REQUIRE(metadata.neighbourExtents[ModelMetadata::LEFT] == std::vector<int> { 4, 5 });
93+
REQUIRE(metadata.neighbourHaloStarts[ModelMetadata::LEFT] == std::vector<int> { 6, 6 });
94+
REQUIRE(metadata.neighbourRanks[ModelMetadata::RIGHT].size() == 0);
95+
REQUIRE(metadata.neighbourRanks[ModelMetadata::BOTTOM].size() == 0);
96+
REQUIRE(metadata.neighbourRanks[ModelMetadata::TOP].size() == 0);
97+
} else {
98+
std::cerr << "only valid for 3 ranks" << std::endl;
99+
exit(1);
100+
}
101+
102+
// this metadata is specific to the periodic boundary conditions
103+
if (test_rank == 0) {
104+
// clang-format off
105+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::LEFT] == std::vector<int> { 2 });
106+
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::LEFT] == std::vector<int> { 4 });
107+
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::LEFT] == std::vector<int> { 2 });
108+
109+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::RIGHT].size() == 0);
110+
111+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 1 });
112+
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 7 });
113+
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 28 });
114+
115+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::TOP].size() == 0);
116+
} else if (test_rank == 1) {
117+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::LEFT] == std::vector<int> { 2 });
118+
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::LEFT] == std::vector<int> { 5 });
119+
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::LEFT] == std::vector<int> { 14 });
120+
121+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::RIGHT].size() == 0);
122+
123+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::BOTTOM].size() == 0);
124+
125+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::TOP] == std::vector<int> { 0 });
126+
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::TOP] == std::vector<int> { 7 });
127+
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::TOP] == std::vector<int> { 0 });
128+
} else if (test_rank == 2) {
129+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::LEFT].size() == 0);
130+
131+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::RIGHT] == std::vector<int> { 0, 1 });
132+
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::RIGHT] == std::vector<int> { 4, 5 });
133+
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::RIGHT] == std::vector<int> { 0, 0 });
134+
135+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 2 });
136+
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 3 });
137+
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::BOTTOM] == std::vector<int> { 24 });
138+
139+
REQUIRE(metadata.neighbourRanksPeriodic[ModelMetadata::TOP] == std::vector<int> { 2 });
140+
REQUIRE(metadata.neighbourExtentsPeriodic[ModelMetadata::TOP] == std::vector<int> { 3 });
141+
REQUIRE(metadata.neighbourHaloStartsPeriodic[ModelMetadata::TOP] == std::vector<int> { 0 });
142+
// clang-format on
143+
} else {
144+
std::cerr << "only valid for 3 ranks" << std::endl;
145+
exit(1);
146+
}
147+
}
148+
149+
}

0 commit comments

Comments
 (0)