Allow tags in name template regexp. (#404)

* Allow tags in name template regexp.

* Add tests.

* Update documentation.

Author: JoeZiminski

datashuttle/utils/formatting.py

@@ -237,13 +237,28 @@ def update_names_with_datetime(names: List[str]) -> None:
     Format using key-value pair for bids, i.e. date-20221223_time-
     """
     date = str(datetime.datetime.now().date().strftime("%Y%m%d"))
-    date_with_key = f"date-{date}"
+    date_with_key = format_date(date)
 
     time_ = datetime.datetime.now().time().strftime("%H%M%S")
-    time_with_key = f"time-{time_}"
+    time_with_key = format_time(time_)
 
-    datetime_with_key = f"datetime-{date}T{time_}"
+    datetime_with_key = format_datetime(date, time_)
 
+    replace_date_time_tags_in_name(
+        names, datetime_with_key, date_with_key, time_with_key
+    )
+
+
+def replace_date_time_tags_in_name(
+    names: List[str],
+    datetime_with_key: str,
+    date_with_key: str,
+    time_with_key: str,
+):
+    """
+    For all names in the list, do the replacement of tags
+    with their final values.
+    """
     for i, name in enumerate(names):
         # datetime conditional must come first.
         if tags("datetime") in name:
@@ -261,6 +276,18 @@ def update_names_with_datetime(names: List[str]) -> None:
             names[i] = name.replace(tags("time"), time_with_key)
 
 
+def format_date(date: str) -> str:
+    return f"date-{date}"
+
+
+def format_time(time_: str) -> str:
+    return f"time-{time_}"
+
+
+def format_datetime(date: str, time_: str) -> str:
+    return f"datetime-{date}T{time_}"
+
+
 def add_underscore_before_after_if_not_there(string: str, key: str) -> str:
     """
     If names are passed with @DATE@, @TIME@, or @DATETIME@

datashuttle/utils/validation.py

@@ -10,7 +10,7 @@
 from itertools import chain
 
 from datashuttle.configs import canonical_folders
-from datashuttle.utils import getters, utils
+from datashuttle.utils import formatting, getters, utils
 from datashuttle.utils.custom_exceptions import NeuroBlueprintError
 
 # -----------------------------------------------------------------------------
@@ -102,6 +102,8 @@ def names_dont_match_templates(
     if regexp is None:
         return False, f"No template set for prefix: {prefix}"
 
+    regexp = replace_tags_in_regexp(regexp)
+
     bad_names = []
     for name in names_list:
         if not re.fullmatch(regexp, name):
@@ -119,6 +121,29 @@ def names_dont_match_templates(
     return False, ""
 
 
+def replace_tags_in_regexp(regexp: str) -> str:
+    """
+    Before validation, all tags in the names are converted to
+    their final values (e.g. @DATE@ -> _date-<date>). We also want to
+    allow template to be formatted like `sub-\d\d_@DATE@` as it
+    is convenient for auto-completion in the TUI.
+
+    Therefore we must replace the tags in the regexp with their
+    actual regexp equivalent before comparison.
+    Note `replace_date_time_tags_in_name()` operates in place on a list.
+    """
+    regexp_list = [regexp]
+    date_regexp = "\d\d\d\d\d\d\d\d"
+    time_regexp = "\d\d\d\d\d\d"
+    formatting.replace_date_time_tags_in_name(
+        regexp_list,
+        datetime_with_key=formatting.format_datetime(date_regexp, time_regexp),
+        date_with_key=formatting.format_date(date_regexp),
+        time_with_key=formatting.format_time(time_regexp),
+    )
+    return regexp_list[0]
+
+
 def get_names_format(bad_names):
     """
     A convenience function to properly format error messages

docs/source/pages/how_tos/use-name-templates.md

@@ -23,6 +23,8 @@ as a regexp where `\d` stands for 'any digit`:
 
 If this is defined as a Name Template, any name that
 does not take this form will result in a validation error.
+Name templates can include [convenience tags](create-folders-convenience-tags).
+(`@DATE@`, `@TIME@` or `@DATETIME@`.)
 
 ## Set up Name Templates
 ::::{tab-set}

tests/tests_integration/test_validation.py

@@ -647,3 +647,57 @@ def test_validate_names_against_project_interactions(self, project):
             "the same ses id as ses-003_id-random. "
             "The existing folder is ses-003." in str(w[1].message)
         )
+
+    def test_tags_in_name_templates_pass_validation(self, project):
+        """
+        It is useful to allow tags in the `name_templates` as it means
+        auto-completion in the TUI can use tags for automatic name
+        generation. Because all subject and session names are
+        fully formatted (e.g. @DATE@ converted to actual dates)
+        prior to validation, the regexp must also have @DATE@
+        and other tags with their regexp equivalent. Check
+        this behaviour here.
+        """
+        name_templates = {
+            "on": True,
+            "sub": "sub-\d\d_@DATE@",
+            "ses": "ses-\d\d\d@DATETIME@",
+        }
+
+        project.set_name_templates(name_templates)
+
+        # Standard behaviour, should not raise
+        project.create_folders(
+            "rawdata",
+            "sub-01_date-20240101",
+            "ses-001_datetime-20240101T142323",
+        )
+        # added tags, should not raise
+        project.create_folders("rawdata", "sub-02@DATE@", "ses-001_@DATETIME@")
+
+        # break the name template validation, for sub, should raise
+        with pytest.raises(NeuroBlueprintError):
+            project.create_folders("rawdata", "sub-03_date_202401")
+
+        # break the name template validation, for ses, should raise
+        with pytest.raises(NeuroBlueprintError):
+            project.create_folders(
+                "rawdata", "sub-03_date_20240101", "ses-001_date-202401"
+            )
+
+        # Do a quick test for tim
+        name_templates["sub"] = "sub-\d\d_@TIME@"
+        project.set_name_templates(name_templates)
+
+        # use time tag, should not raise
+        project.create_folders(
+            "rawdata",
+            "sub-03@TIME@",
+        )
+
+        with pytest.raises(NeuroBlueprintError):
+            # use misspelled time tag, should raise
+            project.create_folders(
+                "rawdata",
+                "sub-03_mime_010101",
+            )

tests/tests_unit/test_validation_unit.py

@@ -225,3 +225,31 @@ def test_new_name_duplicates_existing(self, prefix):
             f"A {prefix} already exists with the same {prefix} id as {prefix}-3. "
             f"The existing folder is {prefix}-3_s-a." in message
         )
+
+    def test_tags_autoreplace_in_regexp(self):
+        """
+        Check the validation function `replace_tags_in_regexp()`
+        correctly replaces tags in a regexp with their regexp equivalent.
+
+        Test date, time and datetime with some random regexp that
+        implicitly check a few other cases (e.g. underscore filling around
+        the tag).
+        """
+        date_regexp = r"sub-\d\d@DATE@_some-tag"
+        fixed_date_regexp = validation.replace_tags_in_regexp(date_regexp)
+        assert fixed_date_regexp == r"sub-\d\d_date-\d\d\d\d\d\d\d\d_some-tag"
+
+        time_regexp = r"ses-\d\d\d\d@TIME@_some-.?.?tag"
+        fixed_time_regexp = validation.replace_tags_in_regexp(time_regexp)
+        assert (
+            fixed_time_regexp == r"ses-\d\d\d\d_time-\d\d\d\d\d\d_some-.?.?tag"
+        )
+
+        datetime_regexp = r"ses-.?.?.?@DATETIME@some-.?.?tag"
+        fixed_datetime_regexp = validation.replace_tags_in_regexp(
+            datetime_regexp
+        )
+        assert (
+            fixed_datetime_regexp
+            == r"ses-.?.?.?_datetime-\d\d\d\d\d\d\d\dT\d\d\d\d\d\d_some-.?.?tag"
+        )