Add motion and emg datatypes

datashuttle/configs/canonical_configs.py

@@ -22,7 +22,6 @@
 
 if TYPE_CHECKING:
     from datashuttle.configs.config_class import Configs
-import copy
 from pathlib import Path
 
 import typeguard
@@ -310,7 +309,8 @@ def get_narrow_datatypes():
     The mapping between broad and narrow datatypes is required for validation.
     """
     return {
-        "ephys": ["ecephys", "icephys"],
+        "behav": ["motion"],
+        "ephys": ["ecephys", "icephys", "emg"],
         "funcimg": ["cscope", "f2pe", "fmri", "fusi"],
         "anat": [
             "2pe",
@@ -350,7 +350,7 @@ def quick_get_narrow_datatypes():
     return flat_narrow_datatypes
 
 
-def in_place_update_settings_for_narrow_datatype(settings: dict):
+def in_place_update_narrow_datatypes_if_required(user_settings: dict):
     """
     In versions < v0.6.0, only 'broad' datatypes were implemented
     and available in the TUI. Since, 'narrow' datatypes are introduced
@@ -359,29 +359,89 @@ def in_place_update_settings_for_narrow_datatype(settings: dict):
 
     This function converts the old format to the new format so that
     all broad datatype settings (on / off) are maintained in
-    then new version.
+    then new version. It does this by copying the full default
+    parameters and overwriting them with the available user-set
+    defaults. This is the best approach, as it maintains the
+    order of the datatypes (otherwise, inserting non-existing
+    datatypes into the user datatype dict results in the wrong order).
+
     """
-    canonical_tui_configs = get_tui_config_defaults()
+    # Find out what is included in the loaded config file,
+    # that determines its version
 
-    new_create_checkbox_configs = copy.deepcopy(
-        canonical_tui_configs["tui"]["create_checkboxes_on"]
-    )
-    new_transfer_checkbox_configs = copy.deepcopy(
-        canonical_tui_configs["tui"]["transfer_checkboxes_on"]
+    has_narrow_datatypes = isinstance(
+        user_settings["tui"]["create_checkboxes_on"]["behav"], dict
+    )  # added 'narrow datatype' v0.6.0 with major refactor to dict
+
+    all_narrow_datatypes = quick_get_narrow_datatypes()
+
+    is_not_missing_any_narrow_datatypes = all(
+        [
+            dtype in user_settings["tui"]["create_checkboxes_on"]
+            for dtype in all_narrow_datatypes
+        ]
     )
 
-    for key in ["behav", "ephys", "funcimg", "anat"]:
-        new_create_checkbox_configs[key]["on"] = settings["tui"][
-            "create_checkboxes_on"
-        ][key]
-        new_transfer_checkbox_configs[key]["on"] = settings["tui"][
-            "transfer_checkboxes_on"
-        ][key]
+    if is_not_missing_any_narrow_datatypes:
+        assert all(
+            [
+                dtype in user_settings["tui"]["transfer_checkboxes_on"]
+                for dtype in all_narrow_datatypes
+            ]
+        ), "Somehow there are datatypes missing in `transfer_checkboxes_on` but not `create_checkboxes_on`"
+
+    if has_narrow_datatypes and is_not_missing_any_narrow_datatypes:
+        return
+
+    # Make a dictionary of the canonical configs to fill in with whatever
+    # user data exists. This ensures the order of the keys is always the same.
+    canonical_tui_configs = get_tui_config_defaults()
 
+    new_checkbox_configs = {
+        "create_checkboxes_on": (
+            canonical_tui_configs["tui"]["create_checkboxes_on"]
+        ),
+        "transfer_checkboxes_on": (
+            canonical_tui_configs["tui"]["transfer_checkboxes_on"]
+        ),
+    }
+
+    # Copy the pre-existing settings unique to the transfer checkboxes
     for key in ["all", "all_datatype", "all_non_datatype"]:
-        new_transfer_checkbox_configs[key]["on"] = settings["tui"][
-            "transfer_checkboxes_on"
-        ][key]
+        if has_narrow_datatypes:
+            new_checkbox_configs["transfer_checkboxes_on"][key] = (
+                user_settings["tui"]["transfer_checkboxes_on"][key]
+            )
+        else:
+            new_checkbox_configs["transfer_checkboxes_on"][key]["on"] = (
+                user_settings["tui"]["transfer_checkboxes_on"][key]
+            )
+
+    # Copy any datatype information that exists. Broad datatypes will all be there
+    # but some narrow datatypes might be missing.
+    for checkbox_type in ["create_checkboxes_on", "transfer_checkboxes_on"]:
+
+        datatypes_that_user_has = list(
+            user_settings["tui"][checkbox_type].keys()
+        )
 
-    settings["tui"]["create_checkboxes_on"] = new_create_checkbox_configs
-    settings["tui"]["transfer_checkboxes_on"] = new_transfer_checkbox_configs
+        for dtype in get_datatypes():
+
+            if dtype in datatypes_that_user_has:
+
+                if has_narrow_datatypes:
+                    new_checkbox_configs[checkbox_type][dtype] = user_settings[
+                        "tui"
+                    ][checkbox_type][dtype]
+                else:
+                    # in versions < 0.6.0 the datatype settings was only a bool
+                    # indicating whether the checkbox is on or not. New versions
+                    # are a dictionary indicating if the checkbox is on ("on")
+                    # and displayed ("displayed").
+                    new_checkbox_configs[checkbox_type][dtype]["on"] = (
+                        user_settings["tui"][checkbox_type][dtype]
+                    )
+
+        user_settings["tui"][checkbox_type] = new_checkbox_configs[
+            checkbox_type
+        ]

datashuttle/datashuttle_class.py

@@ -1551,13 +1551,10 @@ def _update_settings_with_new_canonical_keys(self, settings: Dict):
             if key not in settings["tui"]:
                 settings["tui"][key] = canonical_tui_configs["tui"][key]
 
-        # Handle conversion to 'narrow datatype' v0.6.0
-        if not isinstance(
-            settings["tui"]["create_checkboxes_on"]["behav"], dict
-        ):
-            canonical_configs.in_place_update_settings_for_narrow_datatype(
-                settings
-            )
+        # Handle updating with narrow datatypes
+        canonical_configs.in_place_update_narrow_datatypes_if_required(
+            settings
+        )
 
     def _check_top_level_folder(self, top_level_folder):
         """

datashuttle/tui/screens/datatypes.py

@@ -33,12 +33,14 @@
     "behav": "behaviour",
     "funcimg": "functional imaging",
     "anat": "anatomy",
+    "motion": "motion tracking",
     "ecephys": "extracellular electrophysiology",
     "icephys": "intracellular electrophysiology",
+    "emg": "electromyography",
     "cscope": "head-mounted widefield macroscope",
     "f2pe": "functional 2-photon excitation imaging",
     "fmri": "functional magnetic resonance imaging",
-    "fusi": "functional ultra-sound imaging",
+    "fusi": "functional ultrasound imaging",
     "2pe": "2-photon excitation microscopy",
     "bf": "bright-field microscopy",
     "cars": "coherent anti-Stokes Raman spectroscopy",

tests/test_utils.py

@@ -90,10 +90,9 @@ def glob_basenames(search_path, recursive=False, exclude=None):
 
 
 def teardown_project(
-    cwd, project
+    project,
 ):  # 99% sure these are unnecessary with pytest tmp_path but keep until SSH testing.
     """"""
-    os.chdir(cwd)
     delete_all_folders_in_project_path(project, "central")
     delete_all_folders_in_project_path(project, "local")
     delete_project_if_it_exists(project.project_name)
@@ -109,7 +108,9 @@ def delete_all_folders_in_project_path(project, local_or_central):
     """"""
     folder = f"{local_or_central}_path"
 
-    if folder == "central_path" and project.cfg[folder] is None:
+    if project.cfg is None or (
+        folder == "central_path" and project.cfg[folder] is None
+    ):
         return
 
     ds_logger.close_log_filehandler()
@@ -151,9 +152,7 @@ def setup_project_fixture(tmp_path, test_project_name, project_type="full"):
             local_path=make_test_path(tmp_path, "local", test_project_name)
         )
 
-    cwd = os.getcwd()
-
-    return project, cwd
+    return project
 
 
 def make_test_path(base_path, local_or_central, test_project_name):

tests/tests_integration/base.py

@@ -1,4 +1,3 @@
-import os
 import warnings
 
 import pytest
@@ -58,9 +57,8 @@ def project(self, tmp_path, request):
         else:
             raise ValueError("`parametrized value must be 'full' or 'local'")
 
-        cwd = os.getcwd()
         yield project
-        test_utils.teardown_project(cwd, project)
+        test_utils.teardown_project(project)
 
     @pytest.fixture(scope="function")
     def clean_project_name(self):

tests/tests_integration/test_datatypes.py

@@ -61,8 +61,6 @@ def test_transfer_datatypes(
         """
         subs, sessions = test_utils.get_default_sub_sessions_to_test()
 
-        # Unfortunately on Windows we are encountering 'The command line is too long'
-        # and so cannot test against all datatypes here.
         narrow_datatypes = canonical_configs.quick_get_narrow_datatypes()
 
         datatypes_used = self.get_narrow_only_datatypes_used(used=False)

tests/tests_integration/test_logging.py

@@ -118,7 +118,7 @@ def project(self, tmp_path, clean_project_name, request):
         """
         project_type = getattr(request, "param", "full")
 
-        project, cwd = test_utils.setup_project_fixture(
+        project = test_utils.setup_project_fixture(
             tmp_path, clean_project_name, project_type
         )
 
@@ -128,7 +128,7 @@ def project(self, tmp_path, clean_project_name, request):
 
         yield project
 
-        test_utils.teardown_project(cwd, project)
+        test_utils.teardown_project(project)
         test_utils.set_datashuttle_loggers(disable=True)
 
     # ----------------------------------------------------------------------------------------------------------

tests/tests_integration/test_ssh_file_transfer.py

@@ -80,7 +80,7 @@ def pathtable_and_project(self, request, tmpdir_factory):
             central_path = base_path
         test_project_name = "test_file_conflicts"
 
-        project, cwd = test_utils.setup_project_fixture(
+        project = test_utils.setup_project_fixture(
             base_path, test_project_name
         )
 
@@ -104,7 +104,7 @@ def pathtable_and_project(self, request, tmpdir_factory):
 
         yield [pathtable, project]
 
-        test_utils.teardown_project(cwd, project)
+        test_utils.teardown_project(project)
 
         if testing_ssh:
             for result in glob.glob(ssh_config.FILESYSTEM_PATH):

tests/tests_integration/test_ssh_setup.py

@@ -23,9 +23,7 @@ def project(test, tmp_path):
         tmp_path = tmp_path / "test with space"
 
         test_project_name = "test_ssh"
-        project, cwd = test_utils.setup_project_fixture(
-            tmp_path, test_project_name
-        )
+        project = test_utils.setup_project_fixture(tmp_path, test_project_name)
 
         ssh_test_utils.setup_project_for_ssh(
             project,
@@ -35,7 +33,7 @@ def project(test, tmp_path):
         )
 
         yield project
-        test_utils.teardown_project(cwd, project)
+        test_utils.teardown_project(project)
 
     # -----------------------------------------------------------------
     # Test Setup SSH Connection

tests/tests_regression/old_version_configs/README.md

@@ -0,0 +1,7 @@
+This config and persistent settings are copied from a project created in the GUI
+for the purposes of this regression test. They key information is:
+
+1) The project is called "test_project"
+2) All create and transfer checkboxes are not displayed, except for "f2pe".
+This is so we can check the persistent settings are updated as expected.
+3) The

tests/tests_regression/old_version_configs/v0.5.3/config.yaml

@@ -0,0 +1,5 @@
+local_path: old_ver
+central_path: old_ver
+connection_method: local_filesystem
+central_host_id: null
+central_host_username: null

tests/tests_regression/old_version_configs/v0.5.3/persistent_settings.yaml

@@ -0,0 +1,25 @@
+tui:
+  create_checkboxes_on:
+    behav: false
+    ephys: false
+    funcimg: false
+    anat: false
+  transfer_checkboxes_on:
+    behav: false
+    ephys: false
+    funcimg: false
+    anat: false
+    all: false
+    all_datatype: false
+    all_non_datatype: false
+  top_level_folder_select:
+    create_tab: rawdata
+    toplevel_transfer: rawdata
+    custom_transfer: rawdata
+  bypass_validation: false
+  overwrite_existing_files: never
+  dry_run: false
+name_templates:
+  'on': false
+  sub: null
+  ses: null

tests/tests_regression/old_version_configs/v0.6.0/config.yaml

@@ -0,0 +1,5 @@
+local_path: old_ver
+central_path: old_ver
+connection_method: local_filesystem
+central_host_id: null
+central_host_username: null