From 3b1ee9b8b2f6d984aa3eba13dafaebb8a5b19513 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Wed, 4 Dec 2024 16:45:56 +0000 Subject: [PATCH 1/4] allegedly --- CHANGELOG.md | 10 ++++++++++ python/gputils_api/gputils_api.py | 4 +++- python/test/test.py | 1 + test/testTensor.cu | 23 +++++++++++++++++++---- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 536b867..74ef65f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## v1.7.1 - 4-12-2024 + +### Fixed + +- Compatibility between Python and C++ in how the data is stored in bt files + + diff --git a/python/gputils_api/gputils_api.py b/python/gputils_api/gputils_api.py index e4cc1a1..8e84775 100644 --- a/python/gputils_api/gputils_api.py +++ b/python/gputils_api/gputils_api.py @@ -15,6 +15,7 @@ def read_array_from_gputils_binary_file(path, dt=np.dtype('d')): nm = int.from_bytes(f.read(8), byteorder='little', signed=False) # read number of matrices dat = np.fromfile(f, dtype=np.dtype(dt)) # read data dat = dat.reshape((nr, nc, nm)) # reshape + dat = np.dstack(np.split(dat.reshape(6, -1), 2)) # I'll explain this to you when you grow up return dat @@ -36,8 +37,9 @@ def write_array_to_gputils_binary_file(x, path): nr = x_shape[0] nc = x_shape[1] if x_dims >= 2 else 1 nm = x_shape[2] if x_dims == 3 else 1 + x = np.vstack(np.dsplit(x, 2)).reshape(-1) with open(path, 'wb') as f: f.write(nr.to_bytes(8, 'little')) # write number of rows f.write(nc.to_bytes(8, 'little')) # write number of columns f.write(nm.to_bytes(8, 'little')) # write number of matrices - x.reshape(nr*nc*nm, 1).tofile(f) # write data \ No newline at end of file + x.tofile(f) # write data \ No newline at end of file diff --git a/python/test/test.py b/python/test/test.py index 8c3b6b6..5364191 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -37,6 +37,7 @@ def setUpClass(cls): a = np.linspace(-100, 100, 4 * 5).reshape((4, 5)).astype('d') gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'a_d.bt')) + gpuapi.write_array_to_gputils_binary_file(cls._B, os.path.join(base_dir, 'b_d.bt')) def __test_read_eye(self, dt): diff --git a/test/testTensor.cu b/test/testTensor.cu index fc2e5b7..5af74ac 100644 --- a/test/testTensor.cu +++ b/test/testTensor.cu @@ -182,10 +182,25 @@ TEST_F(TensorTest, parseTensorFromFileBinary) { TEST_F(TensorTest, parseTensorFromBinaryPython) { std::string fName = "../../python/b_d.bt"; - DTensor b = DTensor::parseFromFile(fName); - std::vector vb(12); - b.download(vb); - for (size_t i = 0; i < 12; i++) EXPECT_NEAR(i + 1., vb[i], PRECISION_HIGH); + DTensor b = DTensor::parseFromFile(fName, rowMajor); + for (size_t i=0; i<3; i++) { + for (size_t j=0; j<3; j++) { + EXPECT_NEAR(1 + 2*j + 6*i, b(i, j, 0), PRECISION_HIGH); + EXPECT_NEAR(2 + 2*j + 6*i, b(i, j, 1), PRECISION_HIGH); + } + } +} + + +/* --------------------------------------- + * Parse not existing file + * --------------------------------------- */ + +TEST_F(TensorTest, parseTensorFromNonexistentFile) { + std::string fName = "../../python/whatever.bt"; + EXPECT_THROW(DTensor b = DTensor::parseFromFile(fName, rowMajor), std::invalid_argument); + std::string fName2 = "../../python/whatever.txt"; + EXPECT_THROW(DTensor b = DTensor::parseFromFile(fName2, rowMajor), std::invalid_argument); } From d75cfdf50b19c4814149807046207fa8042b1bb3 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Wed, 4 Dec 2024 17:30:01 +0000 Subject: [PATCH 2/4] it works and dont ask why --- python/gputils_api/gputils_api.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/python/gputils_api/gputils_api.py b/python/gputils_api/gputils_api.py index 8e84775..3fef87f 100644 --- a/python/gputils_api/gputils_api.py +++ b/python/gputils_api/gputils_api.py @@ -14,8 +14,11 @@ def read_array_from_gputils_binary_file(path, dt=np.dtype('d')): nc = int.from_bytes(f.read(8), byteorder='little', signed=False) # read number of columns nm = int.from_bytes(f.read(8), byteorder='little', signed=False) # read number of matrices dat = np.fromfile(f, dtype=np.dtype(dt)) # read data - dat = dat.reshape((nr, nc, nm)) # reshape - dat = np.dstack(np.split(dat.reshape(6, -1), 2)) # I'll explain this to you when you grow up + + if nm >= 2: # if we actually have a 3D tensor (not a matrix or a vector) + dat = dat.reshape((nm, nc, nr)).swapaxes(0, 2) # I'll explain this to you when you grow up + else: + dat = dat.reshape((nr, nc, nm)) # reshape return dat @@ -28,6 +31,7 @@ def write_array_to_gputils_binary_file(x, path): :raises ValueError: if `x` has more than 3 dimensions :raises ValueError: if the file name specified `path` does not have the .bt extension """ + if not path.endswith(".bt"): raise ValueError("The file must have the .bt extension") x_shape = x.shape @@ -37,7 +41,10 @@ def write_array_to_gputils_binary_file(x, path): nr = x_shape[0] nc = x_shape[1] if x_dims >= 2 else 1 nm = x_shape[2] if x_dims == 3 else 1 - x = np.vstack(np.dsplit(x, 2)).reshape(-1) + if x_dims == 3: + x = x.swapaxes(0, 2).reshape(-1) # column-major storage; axis 2 last + else: + x = x.T.reshape(-1) # column-major storage with open(path, 'wb') as f: f.write(nr.to_bytes(8, 'little')) # write number of rows f.write(nc.to_bytes(8, 'little')) # write number of columns From 78881d12daf27b57c0463fde27da95c2a5fcefa5 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Wed, 4 Dec 2024 17:33:26 +0000 Subject: [PATCH 3/4] minor major error --- test/testTensor.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testTensor.cu b/test/testTensor.cu index 5af74ac..6b10636 100644 --- a/test/testTensor.cu +++ b/test/testTensor.cu @@ -182,7 +182,7 @@ TEST_F(TensorTest, parseTensorFromFileBinary) { TEST_F(TensorTest, parseTensorFromBinaryPython) { std::string fName = "../../python/b_d.bt"; - DTensor b = DTensor::parseFromFile(fName, rowMajor); + DTensor b = DTensor::parseFromFile(fName); for (size_t i=0; i<3; i++) { for (size_t j=0; j<3; j++) { EXPECT_NEAR(1 + 2*j + 6*i, b(i, j, 0), PRECISION_HIGH); From 2f3a5f59d30439cd4449efb2d8001309cf32e92e Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Wed, 4 Dec 2024 17:37:19 +0000 Subject: [PATCH 4/4] update version --- python/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/VERSION b/python/VERSION index 9dbb0c0..081af9a 100644 --- a/python/VERSION +++ b/python/VERSION @@ -1 +1 @@ -1.7.0 \ No newline at end of file +1.7.1 \ No newline at end of file