Add support for numpy 2, update all deps (#1139)

Author: rly

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Enhancements
 - Warn when unexpected keys are present in specs. @rly [#1134](https://github.com/hdmf-dev/hdmf/pull/1134)
 - Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136)
+- Add support for numpy 2. @rly [#1139](https://github.com/hdmf-dev/hdmf/pull/1139)
 
 ### Bug fixes
 - Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128)

environment-ros3.yml

@@ -5,11 +5,11 @@ channels:
   - defaults
 dependencies:
   - python==3.12
-  - h5py==3.10.0
-  - matplotlib==3.8.0
-  - numpy==1.26.0
-  - pandas==2.1.2
+  - h5py==3.11.0
+  - matplotlib==3.8.4
+  - numpy==2.0.0
+  - pandas==2.2.2
   - python-dateutil==2.8.2
-  - pytest==7.4.3
-  - pytest-cov==4.1.0
+  - pytest==8.1.2  # regression introduced in pytest 8.2.*, will be fixed in 8.3.0
+  - pytest-cov==5.0.0
   - setuptools

pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
 dependencies = [
     "h5py>=2.10",
     "jsonschema>=2.6.0",
-    'numpy>=1.18, <2.0', # pin below 2.0 until HDMF supports numpy 2.0
+    'numpy>=1.18',
     "pandas>=1.0.5",
     "ruamel.yaml>=0.16",
     "scipy>=1.4",

requirements-dev.txt

@@ -2,12 +2,13 @@
 # compute coverage, and create test environments. note that depending on the version of python installed, different
 # versions of requirements may be installed due to package incompatibilities.
 #
-black==24.3.0
-codespell==2.2.6
-coverage==7.3.2
-pre-commit==3.5.0
-pytest==7.4.3
-pytest-cov==4.1.0
+black==24.4.2
+codespell==2.3.0
+coverage==7.5.4
+pre-commit==3.7.1; python_version >= "3.9"
+pre-commit==3.5.0; python_version < "3.9"
+pytest==8.1.2  # regression introduced in pytest 8.2.*, will be fixed in 8.3.0
+pytest-cov==5.0.0
 python-dateutil==2.8.2
-ruff==0.1.3
-tox==4.11.3
+ruff==0.5.0
+tox==4.15.1

requirements-opt.txt

@@ -1,6 +1,6 @@
 # pinned dependencies that are optional. used to reproduce an entire development environment to use HDMF
-tqdm==4.66.3
-zarr==2.17.1
-linkml-runtime==1.7.4; python_version >= "3.9"
+tqdm==4.66.4
+zarr==2.18.2
+linkml-runtime==1.7.7; python_version >= "3.9"
 schemasheets==0.2.1; python_version >= "3.9"
-oaklib==0.5.32; python_version >= "3.9"
+oaklib==0.6.10; python_version >= "3.9"

requirements.txt

@@ -1,8 +1,10 @@
 # pinned dependencies to reproduce an entire development environment to use HDMF
-h5py==3.10.0
+h5py==3.11.0
 importlib-resources==6.1.0; python_version < "3.9"  # TODO: remove when minimum python version is 3.9
-jsonschema==4.19.1
-numpy==1.26.1
-pandas==2.1.2
+jsonschema==4.22.0
+numpy==1.26.4  # TODO: numpy 2.0.0 is supported by hdmf but incompatible with pandas and scipy
+pandas==2.2.2; python_version >= "3.9"
+pandas==2.1.2; python_version < "3.8"  # TODO: remove when minimum python version is 3.9
 ruamel.yaml==0.18.2
-scipy==1.11.3
+scipy==1.14.0; python_version >= "3.10"
+scipy==1.11.3; python_version < "3.10"

src/hdmf/build/objectmapper.py

@@ -299,7 +299,7 @@ def __check_edgecases(cls, spec, value, spec_dtype):  # noqa: C901
                     cls.__check_convert_numeric(value.dtype.type)
                 if np.issubdtype(value.dtype, np.str_):
                     ret_dtype = 'utf8'
-                elif np.issubdtype(value.dtype, np.string_):
+                elif np.issubdtype(value.dtype, np.bytes_):
                     ret_dtype = 'ascii'
                 elif np.issubdtype(value.dtype, np.dtype('O')):
                     # Only variable-length strings should ever appear as generic objects.

src/hdmf/common/table.py

@@ -235,7 +235,7 @@ def __eq__(self, other):
         if isinstance(search_ids, int):
             search_ids = [search_ids]
         # Find all matching locations
-        return np.in1d(self.data, search_ids).nonzero()[0]
+        return np.isin(self.data, search_ids).nonzero()[0]
 
     def _validate_new_data(self, data):
         # NOTE this may not cover all the many AbstractDataChunkIterator edge cases

tests/unit/test_io_hdf5.py

@@ -121,10 +121,10 @@ def __assert_helper(self, a, b):
                         # if strings, convert before comparing
                         if b_array:
                             if b_sub.dtype.char in ('S', 'U'):
-                                a_sub = [np.string_(s) for s in a_sub]
+                                a_sub = [np.bytes_(s) for s in a_sub]
                         else:
                             if a_sub.dtype.char in ('S', 'U'):
-                                b_sub = [np.string_(s) for s in b_sub]
+                                b_sub = [np.bytes_(s) for s in b_sub]
                         equal = np.array_equal(a_sub, b_sub)
                     else:
                         equal = a_sub == b_sub

tests/unit/utils_test/test_docval.py

@@ -736,8 +736,12 @@ def method(self, **kwargs):
         self.assertEqual(method(self, np.uint(1)), np.uint(1))
         self.assertEqual(method(self, np.uint(2)), np.uint(2))
 
+        # the string rep of uint changes from numpy 1 to 2 ("1" to "np.uint64(1)"), so do not hardcode the string
+        uint_str1 = np.uint(1).__repr__()
+        uint_str2 = np.uint(2).__repr__()
+
         msg = ("TestDocValidator.test_enum_uint.<locals>.method: "
-               "forbidden value for 'arg1' (got 3, expected (1, 2))")
+               "forbidden value for 'arg1' (got 3, expected (%s, %s))" % (uint_str1, uint_str2))
         with self.assertRaisesWith(ValueError, msg):
             method(self, np.uint(3))
 
@@ -767,8 +771,11 @@ def method(self, **kwargs):
         self.assertEqual(method(self, 'true'), 'true')
         self.assertEqual(method(self, np.uint(1)), np.uint(1))
 
+        # the string rep of uint changes from numpy 1 to 2 ("1" to "np.uint64(1)"), so do not hardcode the string
+        uint_str = np.uint(1).__repr__()
+
         msg = ("TestDocValidator.test_enum_bool_mixed.<locals>.method: "
-               "forbidden value for 'arg1' (got 0, expected (True, 1, 1.0, 'true', 1))")
+               "forbidden value for 'arg1' (got 0, expected (True, 1, 1.0, 'true', %s))" % uint_str)
         with self.assertRaisesWith(ValueError, msg):
             method(self, 0)