From fff91110e53ce24a7a46f8f417c08755e59dfbd8 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 2 Dec 2024 19:35:14 +0000 Subject: [PATCH 01/22] python API gputils_api --- python/README.md | 28 +++++++++++++++++ python/VERSION | 1 + python/gputils_api/__init__.py | 1 + python/gputils_api/gputils_api.py | 41 ++++++++++++++++++++++++ python/setup.py | 52 +++++++++++++++++++++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 python/README.md create mode 100644 python/VERSION create mode 100644 python/gputils_api/__init__.py create mode 100644 python/gputils_api/gputils_api.py create mode 100644 python/setup.py diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..ee7928a --- /dev/null +++ b/python/README.md @@ -0,0 +1,28 @@ +## GPUtils API + +### Installation + +As simple as... +```bash +pip install gputils-api +``` +of course, preferably from within a virtual environment. + +### Write to file + +```python + import numpy as np +import gputils_api as g +a = np.eye(3) +g.write_array_to_gputils_binary_file(a, 'my_data.bt') +``` + +### Read from file + +```python +import numpy as np +import gputils_api as g +x = g.read_array_from_gputils_binary_file('my_data.bt') +``` + + diff --git a/python/VERSION b/python/VERSION new file mode 100644 index 0000000..d61f2ad --- /dev/null +++ b/python/VERSION @@ -0,0 +1 @@ +1.7.0rc3 \ No newline at end of file diff --git a/python/gputils_api/__init__.py b/python/gputils_api/__init__.py new file mode 100644 index 0000000..04bdfa9 --- /dev/null +++ b/python/gputils_api/__init__.py @@ -0,0 +1 @@ +from .gputils_api import * \ No newline at end of file diff --git a/python/gputils_api/gputils_api.py b/python/gputils_api/gputils_api.py new file mode 100644 index 0000000..e0a6ae1 --- /dev/null +++ b/python/gputils_api/gputils_api.py @@ -0,0 +1,41 @@ +import numpy as np + +def read_array_from_gputils_binary_file(path, dt=np.dtype('d')): + """ + :param path: path to file + :param dt: numpy-compatible data type + :raises ValueError: if the file name specified `path` does not have the .bt extension + """ + if not path.endswith(".bt"): + raise ValueError("The file must have the .bt extension") + with open(path, 'rb') as f: + nr = int.from_bytes(f.read(8), byteorder='little', signed=False) # read number of rows + nc = int.from_bytes(f.read(8), byteorder='little', signed=False) # read number of columns + nm = int.from_bytes(f.read(8), byteorder='little', signed=False) # read number of matrices + dat = np.fromfile(f, dtype=np.dtype(dt)) # read data + dat = dat.reshape((nr, nc, nm)) # reshape + return dat + + +def write_array_to_gputils_binary_file(x, path): + """ + + :param x: numpy array to save to file + :param path: path to file + :raises ValueError: if `x` has more than 3 dimensions + :raises ValueError: if the file name specified `path` does not have the .bt extension + """ + if not path.endswith(".bt"): + raise ValueError("The file must have the .bt extension") + x_shape = x.shape + x_dims = len(x_shape) + if x_dims >= 4: + raise ValueError("given array cannot have more than 3 dimensions") + nr = x_shape[0] + nc = x_shape[1] if x_dims >= 2 else 1 + nm = x_shape[2] if x_dims == 3 else 1 + with open(path, 'wb') as f: + f.write(nr.to_bytes(8, 'little')) # write number of rows + f.write(nc.to_bytes(8, 'little')) # write number of columns + f.write(nm.to_bytes(8, 'little')) # write number of matrices + x.tofile(f) # write data \ No newline at end of file diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 0000000..2f91c78 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages +import io +import os + +here = os.path.abspath(os.path.dirname(__file__)) + +NAME = 'gputils_api' + +# Import version from file +version_file = open(os.path.join(here, 'VERSION')) +VERSION = version_file.read().strip() + +DESCRIPTION = 'Python API for GPUtils' + + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.md' is present in your MANIFEST.in file! +try: + with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = '\n' + f.read() +except FileNotFoundError: + long_description = DESCRIPTION + +setup(name=NAME, + version=VERSION, + description=DESCRIPTION, + long_description=long_description, + long_description_content_type='text/markdown', + author=['Pantelis Sopasakis', 'Ruairi Moran'], + author_email='p.sopasakis@gmail.com', + license='GNU General Public License v3 (GPLv3)', + packages=find_packages( + exclude=["private"]), + include_package_data=True, + install_requires=[ + 'numpy', 'setuptools' + ], + classifiers=[ + 'Development Status :: 2 - Pre-Alpha ', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python', + 'Environment :: GPU :: NVIDIA CUDA', + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Libraries' + ], + keywords=['api', 'GPU'], + url=( + 'https://github.com/GPUEngineering/GPUtils' + ), + zip_safe=False) From b99d62b8b5886cd3534a113d2b8c117c0fa09148 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 2 Dec 2024 23:06:45 +0000 Subject: [PATCH 02/22] CI: update ci.yml; add python3.12 python/README.md: fix typo --- .github/workflows/ci.yml | 6 +++++- python/README.md | 2 +- python/gputils_api/gputils_api.py | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b686fb..ed03512 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,11 @@ jobs: steps: - name: checkout code uses: actions/checkout@v4 - + - name: setup python3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + architecture: 'x64' - name: run test script run: bash ./ci/script.sh diff --git a/python/README.md b/python/README.md index ee7928a..ebde65a 100644 --- a/python/README.md +++ b/python/README.md @@ -11,7 +11,7 @@ of course, preferably from within a virtual environment. ### Write to file ```python - import numpy as np +import numpy as np import gputils_api as g a = np.eye(3) g.write_array_to_gputils_binary_file(a, 'my_data.bt') diff --git a/python/gputils_api/gputils_api.py b/python/gputils_api/gputils_api.py index e0a6ae1..881999d 100644 --- a/python/gputils_api/gputils_api.py +++ b/python/gputils_api/gputils_api.py @@ -2,6 +2,7 @@ def read_array_from_gputils_binary_file(path, dt=np.dtype('d')): """ + Reads an array from a bt file :param path: path to file :param dt: numpy-compatible data type :raises ValueError: if the file name specified `path` does not have the .bt extension @@ -19,6 +20,7 @@ def read_array_from_gputils_binary_file(path, dt=np.dtype('d')): def write_array_to_gputils_binary_file(x, path): """ + Writes a numpy array into a bt file :param x: numpy array to save to file :param path: path to file From 7198e8621cb35c86a06ca87a3ca47196c4dada95 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 2 Dec 2024 23:10:57 +0000 Subject: [PATCH 03/22] ci script: python and venv --- ci/script.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci/script.sh b/ci/script.sh index 6cf351d..df065d9 100644 --- a/ci/script.sh +++ b/ci/script.sh @@ -17,6 +17,18 @@ tests() { cpp_version=20 fi + + # ------------------------------------ + # Run Python tests first + # ------------------------------------ + pushd python + export PYTHONPATH=. + python -m venv venv + source venv/bin/activate + pip install --upgrade pip + pip install . + popd + # ------------------------------------ # Run tensor gtests # ------------------------------------ From 313d96de7094948a4d498c7863daa4a47ed1d2ca Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 2 Dec 2024 23:15:25 +0000 Subject: [PATCH 04/22] trying with python3.12.4 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed03512..e49b0d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - name: setup python3.12 uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.12.4' architecture: 'x64' - name: run test script run: bash ./ci/script.sh From c451e51db3bc040d79743b5de5365c88d6531269 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 2 Dec 2024 23:17:41 +0000 Subject: [PATCH 05/22] try without python --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e49b0d7..5f88caa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,11 +14,6 @@ jobs: steps: - name: checkout code uses: actions/checkout@v4 - - name: setup python3.12 - uses: actions/setup-python@v5 - with: - python-version: '3.12.4' - architecture: 'x64' - name: run test script run: bash ./ci/script.sh From b6867484e5a1b6974ab4f3997fa642b6a067156e Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 2 Dec 2024 23:35:42 +0000 Subject: [PATCH 06/22] unit test file --- .gitignore | 5 +++++ ci/script.sh | 1 + python/test/test.py | 15 +++++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 python/test/test.py diff --git a/.gitignore b/.gitignore index 16117da..7fe8987 100644 --- a/.gitignore +++ b/.gitignore @@ -260,5 +260,10 @@ paket-files/ # Python Tools for Visual Studio (PTVS) __pycache__/ *.pyc +*.egg-info/ +*/venv/ +*/dist/ +*/build/ + cmake-* \ No newline at end of file diff --git a/ci/script.sh b/ci/script.sh index df065d9..ea98dc4 100644 --- a/ci/script.sh +++ b/ci/script.sh @@ -27,6 +27,7 @@ tests() { source venv/bin/activate pip install --upgrade pip pip install . + python -W ignore test/test.py -v popd # ------------------------------------ diff --git a/python/test/test.py b/python/test/test.py new file mode 100644 index 0000000..9a7a8d0 --- /dev/null +++ b/python/test/test.py @@ -0,0 +1,15 @@ +import unittest + + +class GputilApiTestCase(unittest.TestCase): + + @classmethod + def setUpClass(cls): + pass + + def test_asdf(self): + pass + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 94f7305032ffcd5ad05feb2f577f13a0394813a5 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 2 Dec 2024 23:49:52 +0000 Subject: [PATCH 07/22] test: set up class --- python/test/test.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/python/test/test.py b/python/test/test.py index 9a7a8d0..1ca09ab 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -1,11 +1,22 @@ +import os import unittest +import numpy as np +import gputils_api as gpuapi class GputilApiTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - pass + n = 5 + eye_d = np.eye(n, dtype=np.dtype('d')) + gpuapi.write_array_to_gputils_binary_file(eye_d, 'eye_d.bt') + eye_f = np.eye(n, dtype=np.dtype('f')) + gpuapi.write_array_to_gputils_binary_file(eye_f, 'eye_f.bt') + xd = np.random.randn(2, 4, 6).astype('d') + gpuapi.write_array_to_gputils_binary_file(xd, 'rand_246_d.bt') + xd = np.random.randn(3, 5, 7).astype('f') + gpuapi.write_array_to_gputils_binary_file(xd, 'rand_357_f.bt') def test_asdf(self): pass From b19a974a18787f62036b283fd4b4a4877618c748 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 2 Dec 2024 23:58:44 +0000 Subject: [PATCH 08/22] test: set up class --- python/test/test.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/python/test/test.py b/python/test/test.py index 1ca09ab..c6eb249 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -6,19 +6,27 @@ class GputilApiTestCase(unittest.TestCase): + @staticmethod + def local_abs_path(): + cwd = os.getcwd() + return cwd.split('open-codegen')[0] + @classmethod def setUpClass(cls): n = 5 + base_dir = GputilApiTestCase.local_abs_path() eye_d = np.eye(n, dtype=np.dtype('d')) - gpuapi.write_array_to_gputils_binary_file(eye_d, 'eye_d.bt') + gpuapi.write_array_to_gputils_binary_file(eye_d, os.path.join(base_dir, 'eye_d.bt')) eye_f = np.eye(n, dtype=np.dtype('f')) - gpuapi.write_array_to_gputils_binary_file(eye_f, 'eye_f.bt') + gpuapi.write_array_to_gputils_binary_file(eye_f, os.path.join(base_dir, 'eye_f.bt')) xd = np.random.randn(2, 4, 6).astype('d') - gpuapi.write_array_to_gputils_binary_file(xd, 'rand_246_d.bt') + gpuapi.write_array_to_gputils_binary_file(xd, os.path.join(base_dir, 'rand_246_d.bt')) xd = np.random.randn(3, 5, 7).astype('f') - gpuapi.write_array_to_gputils_binary_file(xd, 'rand_357_f.bt') + gpuapi.write_array_to_gputils_binary_file(xd, os.path.join(base_dir, 'rand_357_f.bt')) + a = np.linspace(-100, 100, 5*6*7).reshape((5, 6, 7)) + gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'rand_357_f.bt')) - def test_asdf(self): + def test_read_eye_d(self): pass From fdd68ebc6226c179f60b9a1715efc311e95fe8db Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 00:02:23 +0000 Subject: [PATCH 09/22] first unit test: test_read_eye_d --- python/test/test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/test/test.py b/python/test/test.py index c6eb249..93e6727 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -27,7 +27,11 @@ def setUpClass(cls): gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'rand_357_f.bt')) def test_read_eye_d(self): - pass + base_dir = GputilApiTestCase.local_abs_path() + path = os.path.join(base_dir, 'eye_d.bt') + r = gpuapi.read_array_from_gputils_binary_file(path, dt=np.dtype('d')) + err = r[:, :, 1] - np.eye(5) + print(err) if __name__ == '__main__': From 35e58dd0296a83102b7cacd639c135e98bf2f35e Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 00:03:12 +0000 Subject: [PATCH 10/22] first unit test: test_read_eye_d --- python/test/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/test/test.py b/python/test/test.py index 93e6727..811669f 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -30,7 +30,7 @@ def test_read_eye_d(self): base_dir = GputilApiTestCase.local_abs_path() path = os.path.join(base_dir, 'eye_d.bt') r = gpuapi.read_array_from_gputils_binary_file(path, dt=np.dtype('d')) - err = r[:, :, 1] - np.eye(5) + err = r[:, :, 0] - np.eye(5) print(err) From 28e1fab9d1f545a316632ae3e17f0736135ce4dc Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 00:07:08 +0000 Subject: [PATCH 11/22] unit tests --- python/test/test.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/python/test/test.py b/python/test/test.py index 811669f..8b39443 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -26,12 +26,19 @@ def setUpClass(cls): a = np.linspace(-100, 100, 5*6*7).reshape((5, 6, 7)) gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'rand_357_f.bt')) - def test_read_eye_d(self): + def __test_read_eye(self, dt): base_dir = GputilApiTestCase.local_abs_path() - path = os.path.join(base_dir, 'eye_d.bt') - r = gpuapi.read_array_from_gputils_binary_file(path, dt=np.dtype('d')) + path = os.path.join(base_dir, f'eye_{dt}.bt') + r = gpuapi.read_array_from_gputils_binary_file(path, dt=np.dtype(dt)) err = r[:, :, 0] - np.eye(5) - print(err) + err_norm = np.linalg.norm(err, np.inf) + self.assertTrue(err_norm < 1e-12) + + def test_read_eye_d(self): + self.__test_read_eye('d') + + def test_read_eye_f(self): + self.__test_read_eye('f') if __name__ == '__main__': From b08024af98c84116a0239d0eda1a3cdd57890118 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 00:13:30 +0000 Subject: [PATCH 12/22] unit tests --- python/test/test.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/python/test/test.py b/python/test/test.py index 8b39443..151efc1 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -20,11 +20,13 @@ def setUpClass(cls): eye_f = np.eye(n, dtype=np.dtype('f')) gpuapi.write_array_to_gputils_binary_file(eye_f, os.path.join(base_dir, 'eye_f.bt')) xd = np.random.randn(2, 4, 6).astype('d') + xd[1, 2, 3] = -123.123 gpuapi.write_array_to_gputils_binary_file(xd, os.path.join(base_dir, 'rand_246_d.bt')) - xd = np.random.randn(3, 5, 7).astype('f') - gpuapi.write_array_to_gputils_binary_file(xd, os.path.join(base_dir, 'rand_357_f.bt')) - a = np.linspace(-100, 100, 5*6*7).reshape((5, 6, 7)) - gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'rand_357_f.bt')) + xf = np.random.randn(2, 4, 6).astype('f') + xf[1, 2, 3] = -123.123 + gpuapi.write_array_to_gputils_binary_file(xf, os.path.join(base_dir, 'rand_246_f.bt')) + a = np.linspace(-100, 100, 5*6*7).reshape((5, 6, 7)).astype('d') + gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'a_d.bt')) def __test_read_eye(self, dt): base_dir = GputilApiTestCase.local_abs_path() @@ -40,6 +42,22 @@ def test_read_eye_d(self): def test_read_eye_f(self): self.__test_read_eye('f') + def __test_read_rand(self, dt): + base_dir = GputilApiTestCase.local_abs_path() + path = os.path.join(base_dir, f'rand_246_{dt}.bt') + r = gpuapi.read_array_from_gputils_binary_file(path, dt=np.dtype(dt)) + r_shape = r.shape + self.assertEqual(2, r_shape[0]) + self.assertEqual(4, r_shape[1]) + self.assertEqual(6, r_shape[2]) + self.assertTrue(np.abs(r[1, 2, 3] + 123.123) < 1e-12) + + def test_read_rand_d(self): + self.__test_read_rand('d') + + def test_read_rand_f(self): + self.__test_read_rand('f') + if __name__ == '__main__': unittest.main() \ No newline at end of file From 1a6cdd0108c327ab54ebf072c7fbaa5e0f51e7db Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 00:42:21 +0000 Subject: [PATCH 13/22] unit tests --- python/test/test.py | 11 ++++++++--- test/testTensor.cu | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/python/test/test.py b/python/test/test.py index 151efc1..d3ea3cf 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -17,14 +17,18 @@ def setUpClass(cls): base_dir = GputilApiTestCase.local_abs_path() eye_d = np.eye(n, dtype=np.dtype('d')) gpuapi.write_array_to_gputils_binary_file(eye_d, os.path.join(base_dir, 'eye_d.bt')) + eye_f = np.eye(n, dtype=np.dtype('f')) gpuapi.write_array_to_gputils_binary_file(eye_f, os.path.join(base_dir, 'eye_f.bt')) + xd = np.random.randn(2, 4, 6).astype('d') - xd[1, 2, 3] = -123.123 + xd[1, 2, 3] = -12.3 gpuapi.write_array_to_gputils_binary_file(xd, os.path.join(base_dir, 'rand_246_d.bt')) + xf = np.random.randn(2, 4, 6).astype('f') - xf[1, 2, 3] = -123.123 + xf[1, 2, 3] = float(-12.3) gpuapi.write_array_to_gputils_binary_file(xf, os.path.join(base_dir, 'rand_246_f.bt')) + a = np.linspace(-100, 100, 5*6*7).reshape((5, 6, 7)).astype('d') gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'a_d.bt')) @@ -50,7 +54,8 @@ def __test_read_rand(self, dt): self.assertEqual(2, r_shape[0]) self.assertEqual(4, r_shape[1]) self.assertEqual(6, r_shape[2]) - self.assertTrue(np.abs(r[1, 2, 3] + 123.123) < 1e-12) + e = np.abs(r[1, 2, 3]+12.3) + self.assertTrue(e < 1e-6) def test_read_rand_d(self): self.__test_read_rand('d') diff --git a/test/testTensor.cu b/test/testTensor.cu index 0e63c2c..6d41ec6 100644 --- a/test/testTensor.cu +++ b/test/testTensor.cu @@ -175,6 +175,23 @@ TEST_F(TensorTest, parseTensorFromFileBinary) { parseTensorFromFileBinary(); } + +/* --------------------------------------- + * Parse files generated by Python + * --------------------------------------- */ + + +TEST_F(TensorTest, parseTensorFromBinaryPython) { + std::string fName = "../../rand_246_d.bt"; + auto a = DTensor::parseFromFile(fName); + EXPECT_EQ(2, a.numRows()); + EXPECT_EQ(4, a.numCols()); + EXPECT_EQ(6, a.numMats()); + auto err = a[1,2,3] + 12.3; + EXPECT_LT(err, 2 * std::numeric_limits::epsilon()); +} + + /* --------------------------------------- * Move constructor * --------------------------------------- */ From f8c749385680f0ade4fcacf240d6dd4497800097 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 00:43:49 +0000 Subject: [PATCH 14/22] unit tests --- test/testTensor.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testTensor.cu b/test/testTensor.cu index 6d41ec6..c56505f 100644 --- a/test/testTensor.cu +++ b/test/testTensor.cu @@ -188,7 +188,7 @@ TEST_F(TensorTest, parseTensorFromBinaryPython) { EXPECT_EQ(4, a.numCols()); EXPECT_EQ(6, a.numMats()); auto err = a[1,2,3] + 12.3; - EXPECT_LT(err, 2 * std::numeric_limits::epsilon()); + EXPECT_LT(err, 2 * std::numeric_limits::epsilon()); } From dfe3895c25f9e05c2ed56aedd8ba33e223df45fa Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 00:45:33 +0000 Subject: [PATCH 15/22] unit tests --- test/testTensor.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testTensor.cu b/test/testTensor.cu index c56505f..be61d8d 100644 --- a/test/testTensor.cu +++ b/test/testTensor.cu @@ -187,7 +187,7 @@ TEST_F(TensorTest, parseTensorFromBinaryPython) { EXPECT_EQ(2, a.numRows()); EXPECT_EQ(4, a.numCols()); EXPECT_EQ(6, a.numMats()); - auto err = a[1,2,3] + 12.3; + auto err = a(1, 2, 3) + 12.3; EXPECT_LT(err, 2 * std::numeric_limits::epsilon()); } From 459164611d14fc8b87fb70891818d07221adbaaf Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 00:52:31 +0000 Subject: [PATCH 16/22] unit tests --- test/testTensor.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testTensor.cu b/test/testTensor.cu index be61d8d..26b51ac 100644 --- a/test/testTensor.cu +++ b/test/testTensor.cu @@ -182,7 +182,7 @@ TEST_F(TensorTest, parseTensorFromFileBinary) { TEST_F(TensorTest, parseTensorFromBinaryPython) { - std::string fName = "../../rand_246_d.bt"; + std::string fName = "../../python/rand_246_d.bt"; auto a = DTensor::parseFromFile(fName); EXPECT_EQ(2, a.numRows()); EXPECT_EQ(4, a.numCols()); From d45cb748c9c2870c0b832fa146533b1c1dfaa3fb Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 01:15:07 +0000 Subject: [PATCH 17/22] unit tests --- python/test/test.py | 8 +++++++- test/testTensor.cu | 10 +++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python/test/test.py b/python/test/test.py index d3ea3cf..b85035a 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -29,9 +29,15 @@ def setUpClass(cls): xf[1, 2, 3] = float(-12.3) gpuapi.write_array_to_gputils_binary_file(xf, os.path.join(base_dir, 'rand_246_f.bt')) - a = np.linspace(-100, 100, 5*6*7).reshape((5, 6, 7)).astype('d') + a = np.linspace(-100, 100, 4*5).reshape((4,5)).astype('d') gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'a_d.bt')) + b = np.array([ + [[1, 2], [3, 4], [5, 6]], + [[7, 8], [9, 10], [-11, 12]] + ], dtype=np.dtype('d')) + gpuapi.write_array_to_gputils_binary_file(b, os.path.join(base_dir, 'b_d.bt')) + def __test_read_eye(self, dt): base_dir = GputilApiTestCase.local_abs_path() path = os.path.join(base_dir, f'eye_{dt}.bt') diff --git a/test/testTensor.cu b/test/testTensor.cu index 26b51ac..0d7eb7a 100644 --- a/test/testTensor.cu +++ b/test/testTensor.cu @@ -182,13 +182,9 @@ TEST_F(TensorTest, parseTensorFromFileBinary) { TEST_F(TensorTest, parseTensorFromBinaryPython) { - std::string fName = "../../python/rand_246_d.bt"; - auto a = DTensor::parseFromFile(fName); - EXPECT_EQ(2, a.numRows()); - EXPECT_EQ(4, a.numCols()); - EXPECT_EQ(6, a.numMats()); - auto err = a(1, 2, 3) + 12.3; - EXPECT_LT(err, 2 * std::numeric_limits::epsilon()); + std::string fName = "../../python/b_d.bt"; + DTensor b = DTensor::parseFromFile(fName); + std::cout << b; } From ed8c4381a86515d2cbd7bb3d55166450997038ad Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 01:29:57 +0000 Subject: [PATCH 18/22] better testing --- python/VERSION | 2 +- python/gputils_api/gputils_api.py | 2 +- python/test/test.py | 2 +- test/testTensor.cu | 5 +++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/python/VERSION b/python/VERSION index d61f2ad..592db39 100644 --- a/python/VERSION +++ b/python/VERSION @@ -1 +1 @@ -1.7.0rc3 \ No newline at end of file +1.7.0rc4 \ No newline at end of file diff --git a/python/gputils_api/gputils_api.py b/python/gputils_api/gputils_api.py index 881999d..e4cc1a1 100644 --- a/python/gputils_api/gputils_api.py +++ b/python/gputils_api/gputils_api.py @@ -40,4 +40,4 @@ def write_array_to_gputils_binary_file(x, path): f.write(nr.to_bytes(8, 'little')) # write number of rows f.write(nc.to_bytes(8, 'little')) # write number of columns f.write(nm.to_bytes(8, 'little')) # write number of matrices - x.tofile(f) # write data \ No newline at end of file + x.reshape(nr*nc*nm, 1).tofile(f) # write data \ No newline at end of file diff --git a/python/test/test.py b/python/test/test.py index b85035a..a2df483 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -34,7 +34,7 @@ def setUpClass(cls): b = np.array([ [[1, 2], [3, 4], [5, 6]], - [[7, 8], [9, 10], [-11, 12]] + [[7, 8], [9, 10], [11, 12]] ], dtype=np.dtype('d')) gpuapi.write_array_to_gputils_binary_file(b, os.path.join(base_dir, 'b_d.bt')) diff --git a/test/testTensor.cu b/test/testTensor.cu index 0d7eb7a..fc2e5b7 100644 --- a/test/testTensor.cu +++ b/test/testTensor.cu @@ -180,11 +180,12 @@ TEST_F(TensorTest, parseTensorFromFileBinary) { * Parse files generated by Python * --------------------------------------- */ - TEST_F(TensorTest, parseTensorFromBinaryPython) { std::string fName = "../../python/b_d.bt"; DTensor b = DTensor::parseFromFile(fName); - std::cout << b; + std::vector vb(12); + b.download(vb); + for (size_t i = 0; i < 12; i++) EXPECT_NEAR(i + 1., vb[i], PRECISION_HIGH); } From 6838ddf2ac98c737eeb2589f73f0124b5c204828 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 01:43:27 +0000 Subject: [PATCH 19/22] better testing --- python/test/test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/test/test.py b/python/test/test.py index a2df483..9f48f21 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -69,6 +69,14 @@ def test_read_rand_d(self): def test_read_rand_f(self): self.__test_read_rand('f') + def test_read_a_tensor_3d(self): + self.__test_read_rand('f') + base_dir = GputilApiTestCase.local_abs_path() + path = os.path.join(base_dir, 'b_d.bt') + b = gpuapi.read_array_from_gputils_binary_file(path) + self.assertEqual(b.shape, (2, 3, 2)) + self.assertAlmostEqual(11., b[1, 2, 0]) + if __name__ == '__main__': unittest.main() \ No newline at end of file From d6635ff812b63135ed5173101ba679eaed452978 Mon Sep 17 00:00:00 2001 From: Ruairi Moran Date: Tue, 3 Dec 2024 14:28:05 +0000 Subject: [PATCH 20/22] squashed --- CMakeLists.txt | 35 +++++++++++++++++------------------ ci/script.sh | 8 ++++---- example/CMakeLists.txt | 4 +--- include/tensor.cuh | 5 +++-- test/CMakeLists.txt | 13 ++++++------- 5 files changed, 31 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b87d7b..9b20dbc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,11 +2,10 @@ # GPUtils # ==================================================================== cmake_minimum_required(VERSION 3.20 FATAL_ERROR) - if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.29") cmake_policy(SET CMP0135 NEW) endif() - +# ---- # Set C++ version and SM architecture if (NOT DEFINED CPPVERSION) set(CPPVERSION 20) # A40: 20, Orin: 17 @@ -14,8 +13,7 @@ endif() if (NOT DEFINED SM_ARCH) set(SM_ARCH 86)# A40: 86, Orin: 87 endif() - - +# ---- project(GPUtils DESCRIPTION "Easy use of vectors and matrices on GPGPU devices." HOMEPAGE_URL "https://github.com/GPUEngineering/GPUtils" @@ -31,8 +29,8 @@ set(CMAKE_CUDA_FLAGS "-std=c++${CPPVERSION}") set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS}; "-std=c++${CPPVERSION}") enable_language(CUDA) # ---- -add_library(device_compiler_flags INTERFACE) -target_compile_features(device_compiler_flags INTERFACE cxx_std_${CPPVERSION}) +add_library(gputils_compiler_flags INTERFACE) +target_compile_features(gputils_compiler_flags INTERFACE cxx_std_${CPPVERSION}) set(CMAKE_CXX_EXTENSIONS OFF) # ---- add_library(developer_flags INTERFACE) @@ -45,30 +43,31 @@ target_compile_options(developer_flags # flags for CUDA builds $<$:${cuda_flags}> ) -target_link_libraries(device_compiler_flags INTERFACE $) +target_link_libraries(gputils_compiler_flags INTERFACE $) # ---- - - -# ==================================================================== -# comment out for release -# ==================================================================== -add_executable(main) -target_sources(main +add_executable(gputils_main) +target_sources(gputils_main PRIVATE main.cu ) -target_link_libraries(main +target_link_libraries(gputils_main PRIVATE - device_compiler_flags + gputils_compiler_flags cublas cusolver cudadevrt ) -target_include_directories(main +target_include_directories(gputils_main PRIVATE "${PROJECT_BINARY_DIR}" "${PROJECT_SOURCE_DIR}/include" ) # ---- -add_subdirectory(test) +if(NOT GPUTILS_BUILD_TEST) + set(GPUTILS_BUILD_TEST OFF) # Set to ON for local testing (or add `-DGPUTILS_BUILD_TEST=ON` to your CMake profile) +endif() +if (GPUTILS_BUILD_TEST) + add_subdirectory(test) +endif() +unset(GPUTILS_BUILD_TEST CACHE) # ---- diff --git a/ci/script.sh b/ci/script.sh index 6cf351d..102976f 100644 --- a/ci/script.sh +++ b/ci/script.sh @@ -7,7 +7,7 @@ tests() { cpp_version=17 # default sm_arch=86 # default hwInfoOrin=`lshw | grep Orin` || - if [ ! -z "${hwInfoOrin}" ]; then + if [ -n "${hwInfoOrin}" ]; then echo "Running on Orin"; sm_arch=87 cpp_version=17 @@ -22,7 +22,7 @@ tests() { # ------------------------------------ # -- create build files - cmake -DCPPVERSION=${cpp_version} -DSM_ARCH=${sm_arch} -S . -B ./build -Wno-dev + cmake -DCPPVERSION=${cpp_version} -DSM_ARCH=${sm_arch} -DGPUTILS_BUILD_TEST=ON -S . -B ./build -Wno-dev # -- build files in build folder cmake --build ./build @@ -34,7 +34,7 @@ tests() { # -- run compute sanitizer pushd ./build/test - mem=$(/usr/local/cuda/bin/compute-sanitizer --tool memcheck --leak-check=full ./device_test) + mem=$(/usr/local/cuda/bin/compute-sanitizer --tool memcheck --leak-check=full ./gputils_test) grep "0 errors" <<< "$mem" popd @@ -44,7 +44,7 @@ tests() { # -- create build files cd example - cmake -DCPPVERSION=${cpp_version} -DSM_ARCH=${sm_arch} -S . -B ./build -Wno-dev + cmake -DCPPVERSION=${cpp_version} -DSM_ARCH=${sm_arch} -S . -B ./build -Wno-dev # -- build files in build folder cmake --build ./build diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 4f84a78..1140d95 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -28,10 +28,8 @@ target_compile_options(example_developer_flags # flags for CUDA builds $<$:${cuda_flags}> ) -target_link_libraries(example_compiler_flags INTERFACE $ -) +target_link_libraries(example_compiler_flags INTERFACE $) # ---- -set(GPUTILS_BUILD_TESTING OFF) include(FetchContent) FetchContent_Declare( gputils diff --git a/include/tensor.cuh b/include/tensor.cuh index cee9dff..bca5706 100644 --- a/include/tensor.cuh +++ b/include/tensor.cuh @@ -608,7 +608,7 @@ data_t vectorFromTextFile(std::string path_to_file) { data_t dataStruct; std::ifstream file; file.open(path_to_file, std::ios::in); - if (!file.is_open()) { throw std::invalid_argument("the file you provided does not exist"); }; + if (!file.is_open()) { throw std::invalid_argument("[vectorFromTextFile] the file does not exist"); } std::string line; getline(file, line); dataStruct.numRows = atoi(line.c_str()); @@ -655,6 +655,7 @@ data_t vectorFromBinaryFile(std::string path_to_file) { /* Read from binary file */ std::ifstream inFile; inFile.open(path_to_file, std::ios::binary); + if (!inFile.is_open()) { throw std::invalid_argument("[vectorFromBinaryFile] the file does not exist"); } inFile.read(reinterpret_cast(&(dataStruct.numRows)), sizeof(uint64_t)); inFile.read(reinterpret_cast(&(dataStruct.numCols)), sizeof(uint64_t)); inFile.read(reinterpret_cast(&(dataStruct.numMats)), sizeof(uint64_t)); @@ -723,7 +724,7 @@ void DTensor::reshape(size_t newNumRows, size_t newNumCols, size_t newNumMats char errMessage[256]; sprintf(errMessage, "DTensor[%lu x %lu x %lu] with %lu elements cannot be reshaped into DTensor[%lu x %lu x %lu] (%lu elements)", - numRows(), numRows(), numMats(), numEl(), newNumRows, newNumCols, newNumMats, newNumElements); + numRows(), numCols(), numMats(), numEl(), newNumRows, newNumCols, newNumMats, newNumElements); throw std::invalid_argument(errMessage); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9259aa9..4abfa1c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -9,24 +9,23 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) # ---- enable_testing() -add_executable(device_test) - -target_sources(device_test # add files +add_executable(gputils_test) +target_sources(gputils_test # add files PRIVATE testTensor.cu ) -target_link_libraries(device_test +target_link_libraries(gputils_test PRIVATE - device_compiler_flags + gputils_compiler_flags cublas cusolver cudadevrt GTest::gtest_main) -target_include_directories(device_test +target_include_directories(gputils_test PRIVATE "${PROJECT_BINARY_DIR}" "${PROJECT_SOURCE_DIR}/include" ) include(GoogleTest) -gtest_discover_tests(device_test) +gtest_discover_tests(gputils_test) # ---- From 8260f3814a96a105f7544f8cb4d7998d5b1f9dde Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 14:30:29 +0000 Subject: [PATCH 21/22] final touch --- ci/script.sh | 1 + python/setup.py | 5 ++++- python/test/test.py | 23 ++++++++++++----------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/ci/script.sh b/ci/script.sh index ea98dc4..4a79739 100644 --- a/ci/script.sh +++ b/ci/script.sh @@ -28,6 +28,7 @@ tests() { pip install --upgrade pip pip install . python -W ignore test/test.py -v + deactivate popd # ------------------------------------ diff --git a/python/setup.py b/python/setup.py index 2f91c78..0495aab 100644 --- a/python/setup.py +++ b/python/setup.py @@ -4,6 +4,9 @@ import io import os +# To publish to pypi, run: +# rm -rf ./build ./dist opengen.egg-info ; pip install . ; python setup.py sdist bdist_wheel; twine upload dist/* + here = os.path.abspath(os.path.dirname(__file__)) NAME = 'gputils_api' @@ -38,7 +41,7 @@ 'numpy', 'setuptools' ], classifiers=[ - 'Development Status :: 2 - Pre-Alpha ', + 'Development Status :: 2 - Pre-Alpha', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 'Programming Language :: Python', 'Environment :: GPU :: NVIDIA CUDA', diff --git a/python/test/test.py b/python/test/test.py index 9f48f21..8c3b6b6 100644 --- a/python/test/test.py +++ b/python/test/test.py @@ -6,6 +6,12 @@ class GputilApiTestCase(unittest.TestCase): + _B = np.array([ + [[1, 2], [3, 4], [5, 6]], + [[7, 8], [9, 10], [11, 12]], + [[13, 14], [15, 16], [17, 18]] + ], dtype=np.dtype('d')) + @staticmethod def local_abs_path(): cwd = os.getcwd() @@ -29,14 +35,9 @@ def setUpClass(cls): xf[1, 2, 3] = float(-12.3) gpuapi.write_array_to_gputils_binary_file(xf, os.path.join(base_dir, 'rand_246_f.bt')) - a = np.linspace(-100, 100, 4*5).reshape((4,5)).astype('d') + a = np.linspace(-100, 100, 4 * 5).reshape((4, 5)).astype('d') gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'a_d.bt')) - - b = np.array([ - [[1, 2], [3, 4], [5, 6]], - [[7, 8], [9, 10], [11, 12]] - ], dtype=np.dtype('d')) - gpuapi.write_array_to_gputils_binary_file(b, os.path.join(base_dir, 'b_d.bt')) + gpuapi.write_array_to_gputils_binary_file(cls._B, os.path.join(base_dir, 'b_d.bt')) def __test_read_eye(self, dt): base_dir = GputilApiTestCase.local_abs_path() @@ -60,7 +61,7 @@ def __test_read_rand(self, dt): self.assertEqual(2, r_shape[0]) self.assertEqual(4, r_shape[1]) self.assertEqual(6, r_shape[2]) - e = np.abs(r[1, 2, 3]+12.3) + e = np.abs(r[1, 2, 3] + 12.3) self.assertTrue(e < 1e-6) def test_read_rand_d(self): @@ -74,9 +75,9 @@ def test_read_a_tensor_3d(self): base_dir = GputilApiTestCase.local_abs_path() path = os.path.join(base_dir, 'b_d.bt') b = gpuapi.read_array_from_gputils_binary_file(path) - self.assertEqual(b.shape, (2, 3, 2)) - self.assertAlmostEqual(11., b[1, 2, 0]) + self.assertEqual(GputilApiTestCase._B.shape, b.shape) + self.assertLess(np.linalg.norm((b - GputilApiTestCase._B).reshape(-1, 1)), 1e-6) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From ea332388a05754f0f92cf857d23f12863c096b18 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 3 Dec 2024 15:17:22 +0000 Subject: [PATCH 22/22] update CHANGELOG and VERSION --- CHANGELOG.md | 18 ++++++++++++++++++ python/VERSION | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 447a7f7..536b867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## v1.7.0 - 3-12-2024 + +### Added + +- The project now uses the GNU General Public License (GPL) v3; license file added +- Introduces new Python package for storing and loading numpy arrays; in can be installed with `pip install gputils-api`; + unit tests and documentation + +### Fixed + +- When compiling with `cmake`, the unit tests will not be compiled by default unless the flag `GPUTILS_BUILD_TEST` is set +- Clang clippy recommendations applied +- Proper error handling when binary tensor file is not found + + diff --git a/python/VERSION b/python/VERSION index 592db39..9dbb0c0 100644 --- a/python/VERSION +++ b/python/VERSION @@ -1 +1 @@ -1.7.0rc4 \ No newline at end of file +1.7.0 \ No newline at end of file