Significant refactoring of grdata.py

I also added documentation and integrated the tool into the testing
framework.

Author: mabruzzo

doc/source/Python.rst

@@ -55,6 +55,17 @@ If your build directory was located in the root level of the grackle repository
 .. note:: Pygrackle can only be run when Grackle is compiled without OpenMP.
    See :ref:`openmp`.
 
+Installing DataFiles
+++++++++++++++++++++
+
+To install the datafiles in a location usable for automatic usage in the Pygrackle examples (and tests) we recommend invoking the following command (from any directory):
+
+.. code-block:: shell-session
+
+   $ python -m pygrackle fetch
+
+:ref:`This section <manage-data-files>` for more details about customizing the the location where data is stored and about managing datafiles in general.
+
 .. _pygrackle-dev:
 
 Installing Pygrackle Development Requirements

doc/source/Tools.rst

@@ -0,0 +1,75 @@
+
+.. _manage-data-files:
+
+Datafile Management
+===================
+
+We provide a command line tool to optionally manage Grackle's datafiles.
+
+At a Quick Glance
+-----------------
+
+Currently, this command line tool is only accessible when :ref:`pygrackle is installed <install-pygrackle>`.
+To execute the tool execute
+
+.. code-block:: shell-session
+
+   $ python -m pygrackle <args>...
+
+Where ``<args>...`` is replaced with one or more command-line arguments.
+For example, ``fetch`` will invoke a subcommand that downloads all associated files (if they aren't already downloaded).
+You can use the ``--help`` option to get a list of all subcommands.
+You can also pass the ``--help`` option after the name of a subcommand (e.g. you can use ``fetch --help``) to get more details about subcommand-specific options.
+
+.. note::
+
+   At the moment, this functionality is most useful for pygrackle.
+   In the near future [#df1]_\ , it will be possible install pygrackle without manually downloading the grackle repository.
+   At that time, this will be the most efficient way to retrieve the files.
+   The pygrackle examples and some of the pygrackle tests rely upon this functionality.
+   However, you are free to completely ignore this functionality for your own purposes.
+
+   There is ongoing work to implement functionality for the Grackle C library to directly access the datafiles managed by this tool.
+   When these efforts are finished, we plan to additionally provide this command-line-tool as a standalone program that is always installed alongside Grackle (so that you can access this functionality without installing pygrackle)
+
+Description
+-----------
+
+.. include:: ../../src/python/pygrackle/utilities/grdata.py
+   :start-after: [[[BEGIN-SECTION:DESCRIPTION]]]
+   :end-before: [[[END-SECTION:DESCRIPTION]]]
+
+
+
+Motivation
+----------
+
+.. include:: ../../src/python/pygrackle/utilities/grdata.py
+   :start-after: [[[BEGIN-SECTION:MOTIVATION]]]
+   :end-before: [[[END-SECTION:MOTIVATION]]]
+
+
+How it works
+------------
+
+.. include:: ../../src/python/pygrackle/utilities/grdata.py
+   :start-after: [[[BEGIN-SECTION:INTERNALS-OVERVIEW]]]
+   :end-before: [[[END-SECTION:INTERNALS-OVERVIEW]]]
+
+
+Sample Directory Structure
+++++++++++++++++++++++++++
+
+Down below, we sketch out what the directory-structure might look like:
+
+
+.. literalinclude:: ../../src/python/pygrackle/utilities/grdata.py
+   :language: none
+   :start-after: [[[BEGIN:DIRECTORY-CARTOON]]]
+   :end-before: [[[END:DIRECTORY-CARTOON]]]
+
+
+.. rubric:: Footnotes
+
+.. [#df1] Once `GH-#208 <https://github.com/grackle-project/grackle/pull/208>`__ is merged, you will be able to instruct pip to install pygrackle by just specifying the URL of the GitHub repository.
+          We also have plans to upload pygrackle to pip.

doc/source/index.rst

@@ -47,6 +47,7 @@ Contents:
    Reference.rst
    Versioning.rst
    Python.rst
+   Tools.rst
    Conduct.rst
    Contributing.rst
    Help.rst

src/python/pygrackle/file_registry/file_registry.txt

@@ -6,10 +6,10 @@
 //    -> ``<outpath>`` with a path to the output file
 //    -> ``<dir>`` with a path to the directory containing all files that are
 //       to be included in the registry
-{"CloudyData_UVB=FG2011.h5", "5b3423fb5cb96d6f8fae65655e204f1f82a276fa"},
-{"CloudyData_UVB=FG2011_shielded.h5", "60d13b4632f074fcb295f7adea85843046c0d4ef"},
-{"CloudyData_UVB=HM2012.h5", "3ae95f71926aa9543964fbd41c5e53a42345c19c"},
-{"CloudyData_UVB=HM2012_high_density.h5", "6db93abf8cb818975e8d751776328c5dab44d4ee"},
-{"CloudyData_UVB=HM2012_shielded.h5", "16cab5b5bd0bf5ef87db717dd5e8901be11812c2"},
-{"CloudyData_noUVB.h5", "55fed7c4bfd10e35d60660ca1adc5ceb411befb2"},
-{"cloudy_metals_2008_3D.h5", "ade563216d1102e8befab822cbb60c418b130aa1"}
+{"CloudyData_UVB=FG2011.h5", "sha1:5b3423fb5cb96d6f8fae65655e204f1f82a276fa"},
+{"CloudyData_UVB=FG2011_shielded.h5", "sha1:60d13b4632f074fcb295f7adea85843046c0d4ef"},
+{"CloudyData_UVB=HM2012.h5", "sha1:3ae95f71926aa9543964fbd41c5e53a42345c19c"},
+{"CloudyData_UVB=HM2012_high_density.h5", "sha1:6db93abf8cb818975e8d751776328c5dab44d4ee"},
+{"CloudyData_UVB=HM2012_shielded.h5", "sha1:16cab5b5bd0bf5ef87db717dd5e8901be11812c2"},
+{"CloudyData_noUVB.h5", "sha1:55fed7c4bfd10e35d60660ca1adc5ceb411befb2"},
+{"cloudy_metals_2008_3D.h5", "sha1:ade563216d1102e8befab822cbb60c418b130aa1"}

src/python/pygrackle/utilities/data_path.py

@@ -11,14 +11,21 @@
 # software.
 ########################################################################
 
-from functools import partial
+import io
 import os
 import sys
 
 from pygrackle.grackle_wrapper import get_grackle_version
-from pygrackle.utilities.grdata import make_config_objects, get_version_dir
+from pygrackle.utilities.grdata import (
+    make_config_objects,
+    VersionDataManager,
+    _parse_file_registry,
+)
 from pygrackle.utilities.misc import dirname
 
+# maybe it would be better to export nothing?
+__all__ = ["grackle_data_dir"]
+
 
 # when we shift to scikit-build-core we can do something more robust here
 def _is_editable_install():
@@ -55,4 +62,13 @@ def _get_file_registry_contents(editable_install):
     file_registry_file=_get_file_registry_contents(is_editable_install),
 )
 
-grackle_data_dir = get_version_dir(*_CONFIG_PAIR)
+_MANAGER = VersionDataManager.create(*_CONFIG_PAIR)
+
+
+def _download_all_datafiles():
+    """Download all datafiles if it hasn't been downloaded already."""
+    registry = _parse_file_registry(_CONFIG_PAIR[1].file_registry_file)
+    return _MANAGER.fetch_all(registry)
+
+
+grackle_data_dir = _MANAGER.version_dir