Merge pull request #59 from janelia-cellmap/rhoadesScholar_update

mzouink · web-flow · commit 76f4b9850210 · 2025-07-22T18:31:57.000+02:00
diff --git a/cellmap_flow/cli/multiple_cli.py b/cellmap_flow/cli/multiple_cli.py
@@ -71,7 +71,7 @@ def main():
         logger.error(
             "Example: cellmap_flow_multiple --data-path /some/shared/path --dacapo -r run_1 -it 60 --dacapo -r run_2 -it 50 --script -s /path/to/script"
         )
-        logger.error("Now we will just open the raw data ..")
+        logger.error("Now we will just open the raw data ...")
 
     # Extract data path
     data_path = None
diff --git a/cellmap_flow/dashboard/app.py b/cellmap_flow/dashboard/app.py
@@ -27,11 +27,19 @@
 import time
 
 logger = logging.getLogger(__name__)
-app = Flask(__name__)
+# Explicitly set template and static folder paths for package installation
+template_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates")
+static_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static")
+app = Flask(__name__, template_folder=template_dir, static_folder=static_dir)
 CORS(app)
 NEUROGLANCER_URL = None
 INFERENCE_SERVER = None
-CustomCodeFolder = "/Users/zouinkhim/Desktop/cellmap/cellmap-flow/example/example_norm"
+CUSTOM_CODE_FOLDER = os.path.expanduser(
+    os.environ.get(
+        "CUSTOM_CODE_FOLDER",
+        "~/Desktop/cellmap/cellmap-flow/example/example_norm",
+    )
+)
 
 
 @app.route("/")
@@ -147,7 +155,7 @@ def process():
             # Save custom code to a file with date and time
             timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
             filename = f"custom_code_{timestamp}.py"
-            filepath = os.path.join(CustomCodeFolder, filename)
+            filepath = os.path.join(CUSTOM_CODE_FOLDER, filename)
 
             with open(filepath, "w") as file:
                 file.write(custom_code)
diff --git a/cellmap_flow/dashboard/templates/index.html b/cellmap_flow/dashboard/templates/index.html
@@ -1,55 +1,46 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
   <meta charset="UTF-8" />
   <title>CellMap Flow Dashboard</title>
   <!-- Bootstrap 5 (optional, for styling) -->
-  <link
-    href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css"
-    rel="stylesheet"
-  />
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet" />
   <link rel="icon" href="{{ url_for('static', filename='img/favicon.ico') }}" type="image/x-icon" />
   <link rel="stylesheet" href="{{ url_for('static', filename='css/dark.css') }}" />
 </head>
+
 <body>
   <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
     <div class="container-fluid">
-      <img 
-      src="https://raw.githubusercontent.com/janelia-cellmap/cellmap-flow/refs/heads/main/img/CMFLOW_dark.png"
-      alt="CellMap Flow"
-      width="200"
-    />
-      <button
-        class="btn btn-outline-light"
-        type="button"
-        onclick="toggleDashboard()"
-      >
+      <img src="https://raw.githubusercontent.com/janelia-cellmap/cellmap-flow/refs/heads/main/img/CMFLOW_dark.png"
+        alt="CellMap Flow" width="200" />
+      <button class="btn btn-outline-light" type="button" onclick="toggleDashboard()">
         Toggle Dashboard
       </button>
     </div>
   </nav>
 
   <div class="row bg-dark text-light" style="height: 100vh;">
     <div class="col-9" id="iframe-column">
-      <iframe src="{{ neuroglancer_url }}" style="width:100%;height:100vh;" title="Example IFrame" id="my_iframe">
+      <iframe src="{{ neuroglancer_url }}" style="width:100%;height:100vh;" title="Neuroglancer Data Viewer"
+        id="my_iframe">
         Your browser does not support iframes.
       </iframe>
     </div>
     <div class="col-3" id="dashboard-column">
       {% include "_dashboard.html" %}
     </div>
-  
+
   </div>
 
-<script
-  src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"
-></script>
-<script src="{{ url_for('static', filename='js/dashboard_script.js') }}"></script>
-<script src="https://cdn.jsdelivr.net/npm/skulpt@1.2.0/dist/skulpt.min.js"></script>
-<script src="https://cdn.jsdelivr.net/npm/skulpt@1.2.0/dist/skulpt-stdlib.js"></script>
-<!-- shared_submit.js or inside a <script> tag in your base layout -->
+  <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+  <script src="{{ url_for('static', filename='js/dashboard_script.js') }}"></script>
+  <script src="https://cdn.jsdelivr.net/npm/skulpt@1.2.0/dist/skulpt.min.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/skulpt@1.2.0/dist/skulpt-stdlib.js"></script>
+  <!-- shared_submit.js or inside a <script> tag in your base layout -->
   <script>
-    document.addEventListener('keydown', function(event) {
+    document.addEventListener('keydown', function (event) {
       // Check if the key pressed is "Enter"
       if (event.key === "Enter") {
         // Trigger a click event on the button
@@ -59,27 +50,27 @@
     document.addEventListener("DOMContentLoaded", function () {
       // Find all elements with id="submitAll" (there could be 2 if both partials exist)
       const submitAllButtons = document.querySelectorAll("#submitAll");
-    
+
       // A single function that merges data & sends the fetch
       function handleSubmitAll() {
         // 1) Gather data from input_norm partial
         const inputNorm = (window.gatherInputNormData)
           ? window.gatherInputNormData()
           : {};
-    
+
         // 2) Gather data from postprocess partial
         const postprocess = (window.gatherPostProcessData)
           ? window.gatherPostProcessData()
           : {};
-    
+
         // 3) Build final combined payload
         const finalPayload = {
           input_norm: inputNorm,
           postprocess: postprocess,
         };
-    
+
         console.log("Combined Payload:", finalPayload);
-    
+
         // 4) Submit via fetch
         fetch("/api/process", {
           method: "POST",
@@ -94,7 +85,7 @@
             if (logAreaNorm) {
               logAreaNorm.value += "Server response:\n" + JSON.stringify(data, null, 2) + "\n";
             }
-    
+
             const logAreaPost = document.getElementById("submissionLog_postProcess");
             if (logAreaPost) {
               logAreaPost.value += "Server response:\n" + JSON.stringify(data, null, 2) + "\n";
@@ -105,13 +96,14 @@
             alert("Error submitting combined data");
           });
       }
-    
+
       // Attach the same handler to each #submitAll
       submitAllButtons.forEach((btn) => {
         btn.addEventListener("click", handleSubmitAll);
       });
     });
-    </script>
-    
+  </script>
+
 </body>
+
 </html>
diff --git a/cellmap_flow/globals.py b/cellmap_flow/globals.py
@@ -29,11 +29,15 @@ def __new__(cls):
             cls._instance.postprocess = []
             cls._instance.viewer = None
             cls._instance.dataset_path = None
-            cls._instance.model_catalog = {}
+            # cls._instance.model_catalog = {}
             # Uncomment and adjust if you want to load the model catalog:
-            # cls._instance.model_catalog = load_model_paths(
-            #     os.path.normpath(os.path.join(os.path.dirname(__file__), os.pardir, "models", "models.yaml"))
-            # )
+            cls._instance.model_catalog = load_model_paths(
+                os.path.normpath(
+                    os.path.join(
+                        os.path.dirname(__file__), os.pardir, "models", "models.yaml"
+                    )
+                )
+            )
             cls._instance.queue = "gpu_h100"
             cls._instance.charge_group = "cellmap"
             cls._instance.neuroglancer_thread = None
diff --git a/cellmap_flow/image_data_interface.py b/cellmap_flow/image_data_interface.py
@@ -1,8 +1,14 @@
+import os
+import zarr
 from cellmap_flow.utils.ds import (
+    find_closest_scale,
     get_ds_info,
     open_ds_tensorstore,
     to_ndarray_tensorstore,
 )
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class ImageDataInterface:
@@ -16,12 +22,19 @@ def __init__(
         concurrency_limit=1,
         normalize=True,
     ):
+        # if multiscale dataset, get scale for voxel size
+        if not isinstance(zarr.open(dataset_path, mode="r"), zarr.core.Array):
+            scale, _, _ = find_closest_scale(dataset_path, voxel_size)
+            logger.info(f"found scale {scale} for voxel size {voxel_size}")
+            dataset_path = os.path.join(dataset_path, scale)
+            logger.info(f"using dataset path {dataset_path}")
         self.path = dataset_path
         self.filetype = (
             "zarr" if dataset_path.rfind(".zarr") > dataset_path.rfind(".n5") else "n5"
         )
         self.swap_axes = self.filetype == "n5"
         self._ts = None
+
         self.voxel_size, self.chunk_shape, self.shape, self.roi, self.swap_axes = (
             get_ds_info(dataset_path)
         )
diff --git a/cellmap_flow/norm/input_normalize.py b/cellmap_flow/norm/input_normalize.py
@@ -182,7 +182,7 @@ def __init__(self, mean=0.0, std=1.0):
     def dtype(self):
         return np.float32
 
-    def normalize(self, data: np.ndarray) -> np.ndarray:
+    def _process(self, data: np.ndarray) -> np.ndarray:
         return (data - self.mean) / self.std
 
 
diff --git a/cellmap_flow/utils/data.py b/cellmap_flow/utils/data.py
@@ -66,6 +66,13 @@ def _get_config(self):
         from cellmap_flow.utils.load_py import load_safe_config
 
         config = load_safe_config(self.script_path)
+        if not hasattr(config, "block_shape"):
+            setattr(
+                config,
+                "block_shape",
+                np.array(tuple(config.write_shape) + (config.output_channels,)),
+            )
+
         return config
 
 
diff --git a/cellmap_flow/utils/ds.py b/cellmap_flow/utils/ds.py
@@ -1,26 +1,20 @@
 # %%
-import zarr
-from funlib.geometry import Coordinate
+import json
 import logging
-import tensorstore as ts
-import numpy as np
-from funlib.geometry import Coordinate
-from funlib.geometry import Roi
 import os
 import re
-import zarr
-from skimage.measure import block_reduce
-from funlib.geometry import Coordinate, Roi
+from typing import Sequence, Union
 
+import h5py
+import numpy as np
+import s3fs
+import tensorstore as ts
 import zarr
+from funlib.geometry import Coordinate, Roi
+from skimage.measure import block_reduce
 from zarr.n5 import N5FSStore
-import h5py
-import json
-import logging
-import os
-from typing import Union, Sequence
+
 from cellmap_flow.globals import g
-import s3fs
 
 
 def get_scale_info(zarr_grp):
@@ -49,6 +43,26 @@ def find_target_scale(zarr_grp_path, target_resolution):
     return target_scale, offsets[target_scale], shapes[target_scale]
 
 
+def find_closest_scale(zarr_grp_path, target_resolution):
+    zarr_grp = zarr.open(zarr_grp_path, mode="r")
+    offsets, resolutions, shapes = get_scale_info(zarr_grp)
+    target_scale = None
+    last_scale = None
+    for scale, res in resolutions.items():
+        if last_scale is None:
+            last_scale = scale
+        if Coordinate(res) == Coordinate(target_resolution):
+            target_scale = scale
+            break
+        elif any((r > t for r, t in zip(res, target_resolution))):
+            target_scale = last_scale
+            break
+        last_scale = scale
+    if target_scale is None:
+        target_scale = last_scale
+    return target_scale, offsets[target_scale], shapes[target_scale]
+
+
 # Ensure tensorstore does not attempt to use GCE credentials
 os.environ["GCE_METADATA_ROOT"] = "metadata.google.internal.invalid"
 
diff --git a/cellmap_flow/utils/load_py.py b/cellmap_flow/utils/load_py.py
@@ -1,7 +1,8 @@
-## copied from https://github.com/janelia-cellmap/cellmap-segmentation-challenge/blob/a9525b31502abb7ea01e10c16340bbc1056cf1fc/src/cellmap_segmentation_challenge/utils/security.py
-
+# copied from https://github.com/janelia-cellmap/cellmap-segmentation-challenge/blob/6e9d842b9a90b0df22aa07946a4d1deed5c27504/src/cellmap_segmentation_challenge/utils/security.py
 import ast
 import os
+import inspect
+from typing import Any
 
 from upath import UPath
 
@@ -67,7 +68,9 @@ def load_safe_config(config_path, force_safe=os.getenv("FORCE_SAFE_CONFIG", Fals
         for issue in issues:
             print(f" - {issue}")
         if force_safe:
-            raise ValueError("Unsafe script detected; loading aborted.")
+            raise ValueError(
+                "Unsafe script detected; loading aborted. You can set the environment variable FORCE_SAFE_CONFIG=False or pass force_safe=False to override."
+            )
 
     # Load the config module if script is safe
     config_path = UPath(config_path)
@@ -76,6 +79,22 @@ def load_safe_config(config_path, force_safe=os.getenv("FORCE_SAFE_CONFIG", Fals
     try:
         with open(config_path, "r") as config_file:
             code = config_file.read()
+            # Parse the code into an AST
+            tree = ast.parse(code)
+
+            # Define a node transformer to replace __file__ with the config path
+            class ReplaceFileNode(ast.NodeTransformer):
+                def visit_Name(self, node):
+                    if node.id == "__file__":
+                        return ast.Constant(value=str(config_path), kind=None)
+                    return node
+
+            # Transform the AST
+            transformer = ReplaceFileNode()
+            tree = transformer.visit(tree)
+
+            # Convert the modified AST back to source code
+            code = ast.unparse(tree)
             exec(code, config_namespace)
         # Extract the config object from the namespace
         config = Config(**config_namespace)
@@ -91,3 +110,39 @@ def load_safe_config(config_path, force_safe=os.getenv("FORCE_SAFE_CONFIG", Fals
 class Config:
     def __init__(self, **kwargs):
         self.__dict__.update(kwargs)
+        self.kwargs = kwargs
+
+    def to_dict(self):
+        """
+        Returns the configuration as a dictionary.
+        """
+        return self.kwargs
+
+    def serialize(self):
+        """
+        Serializes the configuration to a string representation.
+        """
+        serialized = {}
+        for key, value in self.kwargs.items():
+            if (
+                inspect.ismodule(value)
+                or inspect.isclass(value)
+                or inspect.isfunction(value)
+                or inspect.isbuiltin(value)
+            ):
+                # Skip modules, classes, and functions
+                continue
+            elif "__" in key:
+                # Skip private attributes
+                continue
+            elif not isinstance(value, (int, float, str, bool)):
+                serialized[key] = str(value)
+            else:
+                serialized[key] = value
+        return serialized
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """
+        Gets the value of a configuration key.
+        """
+        return self.kwargs.get(key, default)
diff --git a/cellmap_flow/utils/scale_pyramid.py b/cellmap_flow/utils/scale_pyramid.py

Original file line number	Diff line number	Diff line change
`@@ -71,7 +71,7 @@ def main():`
`71`	`71`	`logger.error(`
`72`	`72`	`"Example: cellmap_flow_multiple --data-path /some/shared/path --dacapo -r run_1 -it 60 --dacapo -r run_2 -it 50 --script -s /path/to/script"`
`73`	`73`	`)`
`74`		`- logger.error("Now we will just open the raw data ..")`
	`74`	`+ logger.error("Now we will just open the raw data ...")`
`75`	`75`
`76`	`76`	`# Extract data path`
`77`	`77`	`data_path = None`