Merge develop to main (#308)

* feat: add support dialog

* feat: add support dialog module

* fix: linting and formatting

* Add: footer disclaimer text

* Feat: add trademark disclaimer

* Fix: linting and formatting

* Add: disclaimer to task monitor

* add: padding between task monitor pager and disclaimer window

* add: padding between task monitor pager and disclaimer window

* feat: enabled cloud masking in app component

* fixed collate funcs test

* defined prediction no data val as constant

* Add: disclaimer to task monitor

* add: padding between task monitor pager and disclaimer window

* add: padding between task monitor pager and disclaimer window

* feat: enabled cloud masking in app component

* fixed collate funcs test

* defined prediction no data val as constant

---------

Co-authored-by: Iffanice Houndayi <iffanicehoundayi@gmail.com>

Author: panford-id

instageo/data/raster_chip_creator.py

@@ -33,19 +33,10 @@
 from instageo.data.flags import FLAGS  # Import flags from central location
 from instageo.data.hls_utils import HLSRasterPipeline, add_hls_stac_items
 from instageo.data.s2_utils import S2RasterPipeline, add_s2_stac_items
-from instageo.data.settings import (
-    HLSAPISettings,
-    HLSBandsSettings,
-    HLSBlockSizes,
-    NoDataValues,
-    S2APISettings,
-)
+from instageo.data.settings import HLSAPISettings, S2APISettings
 from instageo.data.stac_utils import create_records_with_items
 
 # Create instances of the settings classes
-NO_DATA_VALUES = NoDataValues()
-HLS_BLOCKSIZE = HLSBlockSizes()
-HLS_BANDS = HLSBandsSettings()
 HLS_API = HLSAPISettings()
 S2_API = S2APISettings()
 

instageo/data/settings.py

@@ -228,3 +228,5 @@ class DataPipelineSettings(BaseSettings):
     BATCH_SIZE: int = 16  # Number of records to process at a time
     METADATA_SEARCH_RATELIMIT: int = 10  # Number of metadata searches per minute
     COG_DOWNLOAD_RATELIMIT: int = 30  # Number of COG downloads per minute
+    HLS_SPATIAL_RESOLUTION: float = 0.0002694945852358564
+    S2_SPATIAL_RESOLUTION: float = 8.983152841195215e-05

instageo/model/configs/config_dataclasses.py

@@ -23,6 +23,7 @@ class ModelInfo(BaseModel):
     num_steps: int = 1
     temporal_step: int = 0
     model_description: str = ""
+    no_data_value: int = 0
 
 
 class ModelEnum(str, Enum):

instageo/model/infer_utils.py

@@ -89,7 +89,7 @@ def chip_inference(
 
     with torch.no_grad():
         with ThreadPoolExecutor(max_workers=num_workers) as executor:
-            for (data, _), file_names in tqdm(dataloader, desc="Running Inference"):
+            for (data, _), file_names, nan_mask in tqdm(dataloader, desc="Running Inference"):
                 data = data.to(device)
                 prediction_batch = model(data)
 
@@ -100,6 +100,10 @@ def chip_inference(
                         torch.argmax(prediction_batch, dim=1).cpu().numpy().astype(np.int8)
                     )
 
+                # Mask out the predictions where the chip had no_data_value
+                nan_mask = np.all(nan_mask, axis=1).astype(int)
+                prediction_batch = np.where(nan_mask == 1, -1, prediction_batch)
+
                 profiles = []
                 for file_name in file_names:
                     with rasterio.open(file_name) as src:
@@ -109,6 +113,7 @@ def chip_inference(
                             dtype=rasterio.int8
                             if prediction_batch.dtype == np.int8
                             else rasterio.float32,
+                            nodata=-1,
                         )
                         profiles.append(profile)
 

instageo/model/pipeline_utils.py

@@ -89,19 +89,22 @@ def eval_collate_fn(batch: tuple[torch.Tensor]) -> tuple[torch.Tensor, torch.Ten
     return data, labels
 
 
-def infer_collate_fn(batch: tuple[torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor]:
+def infer_collate_fn(
+    batch: tuple[torch.Tensor],
+) -> tuple[tuple[torch.Tensor, torch.Tensor], List[str], np.ndarray]:
     """Inference DataLoader Collate Function.
 
     Args:
         batch (Tuple[Tensor]): A list of tuples containing features and labels.
 
     Returns:
-        Tuple of (x,y) concatenated into separate tensors
+        Tuple of ((x,y), filepaths, nan_mask)
     """
     data = torch.stack([a[0][0] for a in batch], 0)
     labels = [a[0][1] for a in batch]
     filepaths = [a[1] for a in batch]
-    return (data, labels), filepaths
+    nan_mask = np.stack([(a[2]) for a in batch], 0)
+    return ((data, labels), filepaths, nan_mask)
 
 
 def create_dataloader(
@@ -282,7 +285,10 @@ def create_instageo_dataset(
         bands=cfg.dataloader.bands,
         replace_label=cfg.dataloader.replace_label,
         reduce_to_zero=cfg.dataloader.reduce_to_zero,
-        chip_no_data_value=cfg.dataloader.no_data_value,
+        chip_no_data_value=max(
+            (cfg.dataloader.no_data_value or 0), 0
+        ),  # backward compatibility with old models where no_data_value was set to -9999/None
+        # no_data_value will be set to 0 for those models
         label_no_data_value=cfg.train.ignore_index,
         constant_multiplier=cfg.dataloader.constant_multiplier,
         include_filenames=include_filenames,

instageo/model/registry/model_registry.py

@@ -64,6 +64,10 @@ def get_model_metadata_for_size(
             num_steps=model_config["dataloader"]["temporal_dim"],
             temporal_step=model_data.get("temporal_step", 0),
             model_description=model_data.get("model_description", "unknown"),
+            no_data_value=max(
+                (model_config["dataloader"].get("no_data_value", 0) or 0), 0
+            ),  # backward compatibility with old models where no_data_value was set to -9999/None
+            # no_data_value will be set to 0 for those models
         )
 
     def get_model_config(

instageo/new_apps/README.md

@@ -173,10 +173,10 @@ Before running the deployment script, you must configure Cloudflare Tunnels:
 Navigate at the root directory of the project and run
 ```bash
 # Deploy with Cloudflare tunnel
-./scripts/deploy.sh
+./scripts/deploy.sh --cloudflare
 
 # Deploy without Cloudflare tunnel
-./scripts/deploy.sh --skip-cloudflare
+./scripts/deploy.sh
 ```
 
 ## Monitoring

instageo/new_apps/backend/app/cog_converter.py

@@ -13,6 +13,8 @@
 
 logger = logging.getLogger(__name__)
 
+PREDICTION_NO_DATA_VALUE = -1
+
 
 class COGConverter:
     """Simple service for merging TIF chips into Cloud Optimized GeoTIFFs using GDAL."""
@@ -22,14 +24,19 @@ def __init__(self) -> None:
         pass
 
     def merge_task_files_to_cog(
-        self, data_path: str, chip_size: int = 256, compute_seg_stats: bool = False
+        self,
+        data_path: str,
+        chip_size: int = 256,
+        compute_seg_stats: bool = False,
+        no_data_value: int = 0,
     ) -> Dict[str, Any]:
         """Merge TIF chips and prediction results from a directory into COGs using GDAL in parallel.
 
         Args:
             data_path: Directory containing files to merge into COG.
             chip_size: Size of chips (used as block size).
             compute_seg_stats: Whether to compute segmentation stats.
+            no_data_value: No-data value to set in COG.
 
         Returns:
             Tuple of (merged_chips_cog_path, merged_predictions_cog_path)
@@ -62,13 +69,15 @@ def merge_task_files_to_cog(
                     str(output_path / "chips_merged.tif"),
                     chip_size,
                     True,  # Enable band selection for chips (keep only B, G, R)
+                    no_data_value,
                 )
                 predictions_future = executor.submit(
                     self.merge_files_to_cog,
                     prediction_files,
                     str(output_path / "predictions_merged.tif"),
                     chip_size,
                     False,  # No band selection for predictions
+                    PREDICTION_NO_DATA_VALUE,
                 )
 
                 # Wait for both to complete and get results
@@ -101,6 +110,7 @@ def merge_files_to_cog(
         output_path: str,
         chip_size: int = 256,
         select_bands: bool = False,
+        no_data_value: int = 0,
     ) -> str:
         """Merge TIF files from a list of files into a single COG using GDAL.
 
@@ -109,6 +119,8 @@ def merge_files_to_cog(
             output_path: Path for output COG file.
             chip_size: Size of chips (used as block size).
             select_bands: If True, keep only first 3 bands (B, G, R) for RGB data.
+            no_data_value: No-data value to set in the COG. If None, the default no-data
+                value will be set.
 
         Returns:
             Path to the created COG file.
@@ -126,6 +138,10 @@ def merge_files_to_cog(
                 "gdal_merge.py",
                 "-o",
                 temp_merged,
+                "-n",
+                str(no_data_value),
+                "-a_nodata",
+                str(no_data_value),
                 *[str(f) for f in tif_files],
             ]
 
@@ -152,6 +168,9 @@ def merge_files_to_cog(
                 )  # Keep only first 3 bands (B, G, R)
                 logger.info("Selecting only first 3 bands (B, G, R) for RGB data")
 
+            cog_cmd.extend(["-a_nodata", str(no_data_value)])
+            logger.info(f"Setting no-data value to {no_data_value} in COG")
+
             cog_cmd.extend(
                 [
                     "-co",

instageo/new_apps/backend/app/data_processor.py

@@ -4,6 +4,7 @@
 satellite data extraction tasks. It handles folder structure, parameter mapping,
 and integration with the task system.
 """
+
 import json
 import logging
 from datetime import datetime
@@ -12,10 +13,12 @@
 
 from absl import flags
 
-from instageo.data.raster_chip_creator import main as bbox_chip_creator
+from instageo.data.settings import DataPipelineSettings
 
 logger = logging.getLogger(__name__)
 
+DATA_PIPELINE_SETTINGS = DataPipelineSettings()
+
 
 class DataProcessor:
     """Proxy class for bounding boxes data pipeline integration."""
@@ -99,6 +102,12 @@ def _prepare_pipeline_params(
 
         params = {
             "is_bbox_feature": True,
+            "spatial_resolution": (
+                DATA_PIPELINE_SETTINGS.HLS_SPATIAL_RESOLUTION
+                if parameters["data_source"] == "HLS"
+                else DATA_PIPELINE_SETTINGS.S2_SPATIAL_RESOLUTION
+            ),
+            "chip_size": parameters["chip_size"],
             "bbox_feature_path": str(bbox_file),
             "output_directory": str(self.data_dir),
             "temporal_tolerance": parameters["temporal_tolerance"],
@@ -108,6 +117,8 @@ def _prepare_pipeline_params(
             "cloud_coverage": parameters["cloud_coverage"],
             "date": parameters["date"],
         }
+        if parameters["mask_cloud"]:
+            params["mask_types"] = "cloud"
         return params
 
     def _run_pipeline(self, params: Dict[str, Any]) -> None:
@@ -116,6 +127,8 @@ def _run_pipeline(self, params: Dict[str, Any]) -> None:
         Args:
             params: Parameters for bounding boxes data pipeline.
         """
+        from instageo.data.raster_chip_creator import main as bbox_chip_creator
+
         # Build command line arguments
         args = ["raster_chip_creator"]
         for key, value in params.items():

instageo/new_apps/backend/app/main.py

@@ -208,6 +208,7 @@ class TaskCreationRequest(BaseModel):
     model_size: str
     date: str
     cloud_coverage: int
+    mask_cloud: bool
     temporal_tolerance: int
 
 
@@ -287,6 +288,7 @@ async def create_task(
             "model_size": model_size,
             "date": task_request.date,
             "cloud_coverage": task_request.cloud_coverage,
+            "mask_cloud": task_request.mask_cloud,
             "temporal_tolerance": task_request.temporal_tolerance or model_info.temporal_step,
             # True model parameters from registry
             "chip_size": model_info.chip_size,
@@ -297,6 +299,7 @@ async def create_task(
             "model_short_name": model_info.model_short_name,
             "model_name": model_info.model_name,
             "classes_mapping": model_info.classes_mapping,
+            "no_data_value": model_info.no_data_value,
         }
 
         # Create task instance (this automatically starts data processing)