New Python package: gputils_api

.github/workflows/ci.yml

@@ -14,7 +14,6 @@ jobs:
     steps:
       - name: checkout code
         uses: actions/checkout@v4
-
       - name: run test script
         run: bash ./ci/script.sh
 

.gitignore

@@ -260,5 +260,10 @@ paket-files/
 # Python Tools for Visual Studio (PTVS)
 __pycache__/
 *.pyc
+*.egg-info/
+*/venv/
+*/dist/
+*/build/
+
 
 cmake-*

ci/script.sh

@@ -17,6 +17,19 @@ tests() {
       cpp_version=20
     fi
 
+
+    # ------------------------------------
+    # Run Python tests first
+    # ------------------------------------
+    pushd python
+    export PYTHONPATH=.
+    python -m venv venv
+    source venv/bin/activate
+    pip install --upgrade pip
+    pip install .
+    python -W ignore test/test.py -v
+    popd
+
     # ------------------------------------
     # Run tensor gtests
     # ------------------------------------

python/README.md

@@ -0,0 +1,28 @@
+## GPUtils API
+
+### Installation
+
+As simple as...
+```bash
+pip install gputils-api
+```
+of course, preferably from within a virtual environment.
+
+### Write to file
+
+```python
+import numpy as np
+import gputils_api as g
+a = np.eye(3)
+g.write_array_to_gputils_binary_file(a, 'my_data.bt')
+```
+
+### Read from file
+
+```python
+import numpy as np
+import gputils_api as g
+x = g.read_array_from_gputils_binary_file('my_data.bt')
+```
+
+

python/VERSION

@@ -0,0 +1 @@
+1.7.0rc4

python/gputils_api/__init__.py

@@ -0,0 +1 @@
+from .gputils_api import *

python/gputils_api/gputils_api.py

@@ -0,0 +1,43 @@
+import numpy as np
+
+def read_array_from_gputils_binary_file(path, dt=np.dtype('d')):
+    """
+    Reads an array from a bt file
+    :param path: path to file
+    :param dt: numpy-compatible data type
+    :raises ValueError: if the file name specified `path` does not have the .bt extension
+    """
+    if not path.endswith(".bt"):
+        raise ValueError("The file must have the .bt extension")
+    with open(path, 'rb') as f:
+        nr = int.from_bytes(f.read(8), byteorder='little', signed=False)  # read number of rows
+        nc = int.from_bytes(f.read(8), byteorder='little', signed=False)  # read number of columns
+        nm = int.from_bytes(f.read(8), byteorder='little', signed=False)  # read number of matrices
+        dat = np.fromfile(f, dtype=np.dtype(dt))  # read data
+        dat = dat.reshape((nr, nc, nm))  # reshape
+    return dat
+
+
+def write_array_to_gputils_binary_file(x, path):
+    """
+    Writes a numpy array into a bt file
+
+    :param x: numpy array to save to file
+    :param path: path to file
+    :raises ValueError: if `x` has more than 3 dimensions
+    :raises ValueError: if the file name specified `path` does not have the .bt extension
+    """
+    if not path.endswith(".bt"):
+        raise ValueError("The file must have the .bt extension")
+    x_shape = x.shape
+    x_dims = len(x_shape)
+    if x_dims >= 4:
+        raise ValueError("given array cannot have more than 3 dimensions")
+    nr = x_shape[0]
+    nc = x_shape[1] if x_dims >= 2 else 1
+    nm = x_shape[2] if x_dims == 3 else 1
+    with open(path, 'wb') as f:
+        f.write(nr.to_bytes(8, 'little'))  # write number of rows
+        f.write(nc.to_bytes(8, 'little'))  # write number of columns
+        f.write(nm.to_bytes(8, 'little'))  # write number of matrices
+        x.reshape(nr*nc*nm, 1).tofile(f)  # write data

python/setup.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+from setuptools import setup, find_packages
+import io
+import os
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+NAME = 'gputils_api'
+
+# Import version from file
+version_file = open(os.path.join(here, 'VERSION'))
+VERSION = version_file.read().strip()
+
+DESCRIPTION = 'Python API for GPUtils'
+
+
+# Import the README and use it as the long-description.
+# Note: this will only work if 'README.md' is present in your MANIFEST.in file!
+try:
+    with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f:
+        long_description = '\n' + f.read()
+except FileNotFoundError:
+    long_description = DESCRIPTION
+
+setup(name=NAME,
+      version=VERSION,
+      description=DESCRIPTION,
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      author=['Pantelis Sopasakis', 'Ruairi Moran'],
+      author_email='p.sopasakis@gmail.com',
+      license='GNU General Public License v3 (GPLv3)',
+      packages=find_packages(
+          exclude=["private"]),
+      include_package_data=True,
+      install_requires=[
+          'numpy', 'setuptools'
+      ],
+      classifiers=[
+          'Development Status :: 2 - Pre-Alpha ',
+          'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+          'Programming Language :: Python',
+          'Environment :: GPU :: NVIDIA CUDA',
+          'Intended Audience :: Developers',
+          'Topic :: Software Development :: Libraries'
+      ],
+      keywords=['api', 'GPU'],
+      url=(
+          'https://github.com/GPUEngineering/GPUtils'
+      ),
+      zip_safe=False)

python/test/test.py

@@ -0,0 +1,82 @@
+import os
+import unittest
+import numpy as np
+import gputils_api as gpuapi
+
+
+class GputilApiTestCase(unittest.TestCase):
+
+    @staticmethod
+    def local_abs_path():
+        cwd = os.getcwd()
+        return cwd.split('open-codegen')[0]
+
+    @classmethod
+    def setUpClass(cls):
+        n = 5
+        base_dir = GputilApiTestCase.local_abs_path()
+        eye_d = np.eye(n, dtype=np.dtype('d'))
+        gpuapi.write_array_to_gputils_binary_file(eye_d, os.path.join(base_dir, 'eye_d.bt'))
+
+        eye_f = np.eye(n, dtype=np.dtype('f'))
+        gpuapi.write_array_to_gputils_binary_file(eye_f, os.path.join(base_dir, 'eye_f.bt'))
+
+        xd = np.random.randn(2, 4, 6).astype('d')
+        xd[1, 2, 3] = -12.3
+        gpuapi.write_array_to_gputils_binary_file(xd, os.path.join(base_dir, 'rand_246_d.bt'))
+
+        xf = np.random.randn(2, 4, 6).astype('f')
+        xf[1, 2, 3] = float(-12.3)
+        gpuapi.write_array_to_gputils_binary_file(xf, os.path.join(base_dir, 'rand_246_f.bt'))
+
+        a = np.linspace(-100, 100, 4*5).reshape((4,5)).astype('d')
+        gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'a_d.bt'))
+
+        b = np.array([
+            [[1, 2], [3, 4], [5, 6]],
+            [[7, 8], [9, 10], [11, 12]]
+        ], dtype=np.dtype('d'))
+        gpuapi.write_array_to_gputils_binary_file(b, os.path.join(base_dir, 'b_d.bt'))
+
+    def __test_read_eye(self, dt):
+        base_dir = GputilApiTestCase.local_abs_path()
+        path = os.path.join(base_dir, f'eye_{dt}.bt')
+        r = gpuapi.read_array_from_gputils_binary_file(path, dt=np.dtype(dt))
+        err = r[:, :, 0] - np.eye(5)
+        err_norm = np.linalg.norm(err, np.inf)
+        self.assertTrue(err_norm < 1e-12)
+
+    def test_read_eye_d(self):
+        self.__test_read_eye('d')
+
+    def test_read_eye_f(self):
+        self.__test_read_eye('f')
+
+    def __test_read_rand(self, dt):
+        base_dir = GputilApiTestCase.local_abs_path()
+        path = os.path.join(base_dir, f'rand_246_{dt}.bt')
+        r = gpuapi.read_array_from_gputils_binary_file(path, dt=np.dtype(dt))
+        r_shape = r.shape
+        self.assertEqual(2, r_shape[0])
+        self.assertEqual(4, r_shape[1])
+        self.assertEqual(6, r_shape[2])
+        e = np.abs(r[1, 2, 3]+12.3)
+        self.assertTrue(e < 1e-6)
+
+    def test_read_rand_d(self):
+        self.__test_read_rand('d')
+
+    def test_read_rand_f(self):
+        self.__test_read_rand('f')
+
+    def test_read_a_tensor_3d(self):
+        self.__test_read_rand('f')
+        base_dir = GputilApiTestCase.local_abs_path()
+        path = os.path.join(base_dir, 'b_d.bt')
+        b = gpuapi.read_array_from_gputils_binary_file(path)
+        self.assertEqual(b.shape, (2, 3, 2))
+        self.assertAlmostEqual(11., b[1, 2, 0])
+
+
+if __name__ == '__main__':
+    unittest.main()

test/testTensor.cu

@@ -175,6 +175,20 @@ TEST_F(TensorTest, parseTensorFromFileBinary) {
     parseTensorFromFileBinary<double>();
 }
 
+
+/* ---------------------------------------
+ * Parse files generated by Python
+ * --------------------------------------- */
+
+TEST_F(TensorTest, parseTensorFromBinaryPython) {
+    std::string fName = "../../python/b_d.bt";
+    DTensor<double> b = DTensor<double>::parseFromFile(fName);
+    std::vector<double> vb(12);
+    b.download(vb);
+    for (size_t i = 0; i < 12; i++) EXPECT_NEAR(i + 1., vb[i], PRECISION_HIGH);
+}
+
+
 /* ---------------------------------------
  * Move constructor
  * --------------------------------------- */