debugging t1w pipeline and associated unittests

Author: tclose

.github/workflows/release.yml

@@ -22,10 +22,10 @@ jobs:
       matrix:
         spec:
           - quality-control/phi-finder
-          - mri/human/neuro/bidsapp/fmriprep
-          - mri/human/neuro/bidsapp/mriqc
-          - mri/human/neuro/bidsapp/smriprep
-          # - mri/human/neuro/t1w/preprocess
+          # - mri/human/neuro/bidsapp/fmriprep
+          # - mri/human/neuro/bidsapp/mriqc
+          # - mri/human/neuro/bidsapp/smriprep
+          - mri/human/neuro/t1w/preprocess
     steps:
     - name: Removed unnecessary tools to free space
       run: |

conftest.py

@@ -8,6 +8,7 @@
 from frametree.core.utils import varname2path
 import pytest
 from click.testing import CliRunner
+import xnat
 import xnat4tests
 
 # Set DEBUG logging for unittests
@@ -70,13 +71,17 @@ class BidsAppTestBlueprint:
     / "neuro"
     / "bidsapps"
 )
-test_bids_data_dir = (
-    Path(__file__).parent / "tests" / "data" / "mri" / "human" / "neuro" / "bidsapps"
-)
+test_data_dir = Path(__file__).parent / "tests" / "data"
+test_bids_data_dir = test_data_dir / "specs" / "mri" / "human" / "neuro" / "bidsapps"
 
 bids_specs = [str(p.stem) for p in bids_apps_dir.glob("*.yaml")]
 
 
+@pytest.fixture(params=bids_specs)
+def bids_app_spec_path(request):
+    return bids_apps_dir / (request.param + ".yaml")
+
+
 @pytest.fixture(params=bids_specs)
 def bids_app_blueprint(run_prefix, xnat_connect, request):
     bids_app_name = request.param
@@ -168,4 +173,10 @@ def upload_test_dataset_to_xnat(project_id: str, source_data_dir: Path, xnat_con
                 xresource.upload_dir(resource_path, method="tar_file")
 
         # Populate metadata from DICOM headers
-        login.put(f"/data/experiments/{xsession.id}?pullDataFromHeaders=true")
+        try:
+            login.put(f"/data/experiments/{xsession.id}?pullDataFromHeaders=true")
+        except xnat.exceptions.XNATResponseError as e:
+            logger.warning(
+                f"Failed to pull metadata from DICOM headers for session {xsession.id} "
+                f"with error: {e}"
+            )

specs/australian-imaging-service/mri/human/neuro/t1w/preprocess.yaml

@@ -9,13 +9,13 @@ base_image:
 packages:
   # Install dependencies for DICOM->NIfTI conversion
   pip:
-    pydra-mrtrix3: 3.0.4a9
-    pydra-fsl:
+    pydra-tasks-mrtrix3:
+    pydra-tasks-fsl:
     fileformats:
-    pydra-fastsurfer:
+    pydra-tasks-fastsurfer:
     fileformats-medimage-extras:
-    fileformats-medimage-mrtrix3: 3.0.4a12
-    fileformats-medimage-mrtrix3-extras: 3.0.4a12
+    fileformats-medimage-mrtrix3:
+    fileformats-medimage-mrtrix3-extras:
     australianimagingservice:
   neurodocker:
     mrtrix3: 3.0.2
@@ -66,31 +66,33 @@ commands:
   single_parc:
     task: australianimagingservice.mri.human.neuro.t1w.preprocess:SingleParcellation
     operates_on: medimage/session
-    inputs:
-       - t1w
+    sources:
+      T1w:
+        field: t1w
+        help: "The T1-weighted MRI dataset to preprocess"
+    parameters:
+      Parcellation:
+        field: parcellation
+        help: "The parcellation to parcelate the cortex with"
     configuration: # Additional args passed to arcana.common:shell_cmd
       mrtrix_lut_dir: /parcellations # /opt/mrtrix3-3.0.2/share/mrtrix3/labelconvert
       freesurfer_home: &freesurfer_home !join ["/opt/freesurfer-", *freesurfer_version]
       fs_license: *freesurfer_license
-      cache_dir: /work/pydra-cache
-      fastsurfer_executable:
-        - /fastsurfer/Docker/entrypoint.sh
-        - /fastsurfer/run_fastsurfer.sh
-        - --allow_root
+      subjects_dir: /work/subjects
+      in_fastsurfer_container: true
   all_parcs:
     task: australianimagingservice.mri.human.neuro.t1w.preprocess:AllParcellations
     operates_on: medimage/session
-    inputs: # List the inputs that are presented to end-user in UI
-      - t1w
+    sources:
+      T1w:
+        field: t1w
+        help: "The T1-weighted MRI dataset to preprocess"
     configuration: # Additional args passed to arcana.common:shell_cmd
       mrtrix_lut_dir: /parcellations # /opt/mrtrix3-3.0.2/share/mrtrix3/labelconvert ## should this be 
-      freesurfer_home: &freesurfer_home !join ["/opt/freesurfer-", *freesurfer_version] ## freesurfer_home: *freesurfer_home
+      freesurfer_home: *freesurfer_home ## freesurfer_home: *freesurfer_home
       fs_license: *freesurfer_license
-      cache_dir: /work/pydra-cache
-      fastsurfer_executable:
-        - /fastsurfer/Docker/entrypoint.sh
-        - /fastsurfer/run_fastsurfer.sh
-        - --allow_root
+      subjects_dir: /work/subjects
+      in_fastsurfer_container: true
 resources:
   neuro-parcellations: /parcellations
   yeo2011-parcellations: !join [*freesurfer_home, "/Yeo2011"]

src/australianimagingservice/mri/human/neuro/t1w/preprocess/all_parcs.py

@@ -71,7 +71,7 @@ def AllParcellations(
     freesurfer_home: Directory,
     mrtrix_lut_dir: Directory,
     fs_license: File,
-    fastsurfer_executable: str | list[str] | None = None,
+    in_fastsurfer_container: bool = False,
     fastsurfer_python: str = "python3",
 ) -> tuple[
     DirectoryOf[Mif],
@@ -96,7 +96,7 @@ def AllParcellations(
                 freesurfer_home=freesurfer_home,
                 mrtrix_lut_dir=mrtrix_lut_dir,
                 fs_license=fs_license,
-                fastsurfer_executable=fastsurfer_executable,
+                in_fastsurfer_container=in_fastsurfer_container,
                 fastsurfer_python=fastsurfer_python,
                 subjects_dir=subjects_dir,
             ),  # pyright: ignore[reportArgumentType]

src/australianimagingservice/mri/human/neuro/t1w/preprocess/single_parc.py

@@ -17,6 +17,7 @@
 from fileformats.medimage import NiftiGz
 from fileformats.medimage_mrtrix3 import ImageFormat as Mif
 from pydra.environments.docker import Docker
+from pydra.environments.native import Native
 from pydra.tasks.fastsurfer.latest import Fastsurfer
 from .helpers import JoinTaskCatalogue, LabelSgmFix, Dependency
 
@@ -46,42 +47,48 @@ def SingleParcellation(
     mrtrix_lut_dir: Directory,
     fs_license: File,
     subjects_dir: Path,
-    fastsurfer_executable: ty.Union[str, ty.List[str], None] = None,
+    in_fastsurfer_container: bool = False,
     fastsurfer_python: str = "python3",
 ) -> tuple[Mif, Mif | None, Mif | None, Mif | None, Mif | None, Mif | None, Mif | None]:
 
     # ###################
     # # FASTSURFER TASK #
     # ###################
 
+    if in_fastsurfer_container:
+        fs_environment = Native()
+    else:
+        fs_environment = Docker(
+            image="deepmi/fastsurfer",
+            tag="cpu-v2.4.2",
+            xargs=[
+                "--user",
+                "1000:1000",
+                "--entrypoint",
+                "/bin/bash",
+            ],
+        )
+
     fastsurfer = workflow.add(
         Fastsurfer(
             T1_files=t1w,
             fs_license=fs_license,
             subject_id="FS_outputs",
-            py=fastsurfer_python,
             # norm_img="norm.mgz",
             # aparcaseg_img="aparcaseg.mgz",
             fsaparc=True,
             parallel=True,
             threads=24,
             subjects_dir=subjects_dir,
         ),
-        environment=Docker(
-            image="deepmi/fastsurfer",
-            tag="cpu-v2.4.2",
-            xargs=[
-                "--user",
-                "1000:1000",
-                "--entrypoint",
-                "/bin/bash",
-                "-v",
-                f"{subjects_dir}:/mnt/pydra/{subjects_dir}:rw",
-            ],
-        ),
+        environment=fs_environment,
     )
-    if fastsurfer_executable:
-        fastsurfer.inputs.executable = fastsurfer_executable
+
+    if in_fastsurfer_container:
+        fastsurfer.inputs.py = "python3"
+        fastsurfer.inputs.executable = "/fastsurfer/run_fastsurfer.sh"
+    else:
+        fastsurfer.inputs.py = fastsurfer_python
 
     # #################################################
     # # FIVE TISSUE TYPE Generation and visualisation #

src/australianimagingservice/mri/human/neuro/t1w/preprocess/tests/test_preprocess.py

@@ -1,41 +1,28 @@
+import os
 from pathlib import Path
 from pydra2app.core import App
 from australianimagingservice.mri.human.neuro.t1w.preprocess import AllParcellations
+import xnat
+import pytest
+from conftest import upload_test_dataset_to_xnat, test_data_dir
 
-PKG_DIR = (
-    Path(__file__).parent / ".." / ".." / ".." / ".." / ".." / ".." / ".."
-).resolve()
+home_dir = Path("~").expanduser()
 
 
-def test_t1w_preprocess():
+@pytest.mark.skip
+def test_t1w_preprocess(tmp_path: Path):
+
+    test_data = test_data_dir / "src" / "mri" / "human" / "neuro" / "t1w" / "preprocess"
+
     wf = AllParcellations(
-        t1w=,
-        subjects_dir=,
-        freesurfer_home=,
-        mrtrix_lut_dir=,
-        fs_license=,
-        "/opt/FastSurfer",
-        "/opt/mrtrix3/3.0.4/share/mrtrix3/labelconvert",
-        cache_dir="/Users/tclose/Desktop/cache-dir",
-        "/Users/tclose/Desktop/file1.txt",
+        t1w=test_data / "t1w.nii.gz",
+        subjects_dir=tmp_path / "subjects",
+        freesurfer_home=os.environ["FREESURFER_HOME"],
+        mrtrix_lut_dir=(
+            Path(os.environ["MRTRIX3_HOME"]) / "share" / "mrtrix3" / "labelconvert"
+        ),
+        fs_license=Path(os.environ["FREESURFER_HOME"]).parent / "licence.txt",
     )
 
     outputs = wf()
     print(outputs)
-
-
-def test_t1w_preprocess_yaml(tmp_path: Path):
-    app = App.load(
-        PKG_DIR
-        / "specs"
-        / "australian-imaging-service"
-        / "mri"
-        / "human"
-        / "neuro"
-        / "t1w"
-        / "preprocess.yaml"
-    )
-
-    app.make(
-        build_dir=tmp_path, generate_only=True, resources_dir=PKG_DIR / "resources"
-    )

tests/test_phi_finder.py

@@ -3,13 +3,15 @@
 from pydra2app.core.cli import make
 from frametree.core.utils import show_cli_trace
 import xnat4tests
+import pytest
 from pydra2app.xnat.deploy import install_cs_command, launch_cs_command
 
 PKG_PATH = Path(__file__).parent.parent.absolute()
 
 runner = CliRunner()
 
 
+@pytest.mark.skip
 def test_phi_finder():
     result = runner.invoke(
         make,

tests/test_t1_preprocess.py

@@ -0,0 +1,101 @@
+import json
+import typing as ty
+from pathlib import Path
+from pydra2app.core.cli import make
+from pydra2app.xnat import XnatApp
+from frametree.core.utils import show_cli_trace
+from pydra2app.xnat.deploy import install_and_launch_xnat_cs_command
+from conftest import upload_test_dataset_to_xnat, test_data_dir
+
+PKG_DIR = Path(__file__).parent.parent
+
+SPEC_PATH = (
+    PKG_DIR
+    / "specs"
+    / "australian-imaging-service"
+    / "mri"
+    / "human"
+    / "neuro"
+    / "t1w"
+    / "preprocess.yaml"
+)
+
+RESOURCES_DIR = PKG_DIR / "resources"
+
+SKIP_BUILD = False
+
+
+def test_t1_preprocess_app(
+    run_prefix: str,
+    xnat_connect: ty.Any,
+    cli_runner: ty.Callable[..., ty.Any],
+    tmp_path: Path,
+):
+
+    build_dir = tmp_path / "build"
+
+    build_dir.mkdir(exist_ok=True, parents=True)
+
+    project_id = f"{run_prefix}mrihumanneurot1wpreprocess"
+    test_data = (
+        test_data_dir / "specs" / "mri" / "human" / "neuro" / "t1w" / "preprocess"
+    )
+    upload_test_dataset_to_xnat(project_id, test_data, xnat_connect)
+
+    if SKIP_BUILD:
+        build_arg = "--generate-only"
+    else:
+        build_arg = "--build"
+
+    result = cli_runner(
+        make,
+        [
+            "xnat",
+            str(SPEC_PATH),
+            "--build-dir",
+            str(build_dir),
+            build_arg,
+            "--resources-dir",
+            str(RESOURCES_DIR),
+            "--for-localhost",
+            "--use-local-packages",
+            "--raise-errors",
+        ],
+    )
+
+    assert result.exit_code == 0, show_cli_trace(result)
+
+    image_spec = XnatApp.load(SPEC_PATH)
+
+    with xnat_connect() as xlogin:
+
+        with open(
+            build_dir / image_spec.name / "xnat_commands" / (image_spec.name + ".json")
+        ) as f:
+            xnat_command = json.load(f)
+        xnat_command.name = xnat_command.label = image_spec.name + run_prefix
+
+        test_xsession = next(
+            iter(xnat_connect.projects[project_id].experiments.values())
+        )
+
+        inputs_json = {
+            "T1w": test_data / "T1w",
+            "Parcellation": "desikan",
+            "pydra2app_flags": (
+                "--worker serial "
+                "--work /work "  # NB: work dir moved inside container due to file-locking issue on some mounted volumes (see https://github.com/tox-dev/py-filelock/issues/147)
+                "--dataset-name default "
+                "--loglevel debug "
+            ),
+        }
+
+        workflow_id, status, out_str = install_and_launch_xnat_cs_command(
+            command_json=xnat_command,
+            project_id=project_id,
+            session_id=test_xsession.id,
+            inputs=inputs_json,
+            xlogin=xnat_connect,
+            timeout=30000,
+        )
+        assert status == "Complete", f"Workflow {workflow_id} failed.\n{out_str}"