Add interactive view of catalog (#723)

* Add interactive view of catalog

* Add itables to environment-upsteam-dev.yml too

* update docs to include interactive catalog demo - should also function as a test - doc failure <=> broken

* Make itables work better:
- Fix columns_with_iterables
- Add widgets for multi filtering

* Explode iterable columns one at a time

* Set cascading panes to closed by default

* Add MinimalExploder to 'jointly explode' compatible iterable columns - this looks for iterable columns which look to be over the same lists & explodes them together

Author: charles-turner-1

ci/environment-docs.yml

@@ -10,6 +10,7 @@ dependencies:
   - fsspec >=2024.12
   - gcsfs >=2024.12
   - intake >=2.0
+  - itables
   - jupyterlab
   - matplotlib
   - myst-nb

ci/environment-upstream-dev.yml

@@ -12,6 +12,7 @@ dependencies:
   - gcsfs >=2024.12
   - h5netcdf >=0.8.1
   - ipython
+  - itables
   - matplotlib
   - netcdf4 >=1.5.5,!=1.6.1
   - pandas >=2.1.0

ci/environment.yml

@@ -13,6 +13,7 @@ dependencies:
   - h5netcdf >=0.8.1
   - intake >=2.0
   - ipython
+  - itables
   - matplotlib
   - netcdf4 >=1.5.5,!=1.6.1
   - pandas >=2.1.0

docs/source/how-to/use-catalogs-with-assets-containing-multiple-variables.md

@@ -63,13 +63,21 @@ cat.esmcat.has_multiple_variable_assets
 
 ## Search for datasets
 
-The search functionatilty works in the same way:
+The search functionality works in the same way:
 
 ```{code-cell} ipython3
 cat_subset =cat.search(variable=["O2", "SiO3"])
 cat_subset.df
 ```
 
+### Interactively search the catalog
+
+We can also use the `interactive` attribute of a catalog to interactively search the catalog. This will not save any searches, but allows you to explore the catalog in a quick and intuitive way.
+
+```{code-cell} ipython3
+cat.interactive
+```
+
 ## Load assets into xarray datasets
 
 When loading the data files into xarray datasets, `intake-esm` will load only **data variables** that were requested. For example, if a data file contains ten data variables and the user requests for two variables, intake-esm will load the two requested variables plus necessary coordinates information.

intake_esm/core.py

@@ -17,14 +17,17 @@
     _DATATREE_AVAILABLE = True
 except ImportError:
     _DATATREE_AVAILABLE = False
+import itables
 import pandas as pd
+import polars as pl
 import pydantic
 from fastprogress.fastprogress import progress_bar
 from intake.catalog import Catalog
 
 from .cat import ESMCatalogModel
 from .derived import DerivedVariableRegistry, default_registry
 from .source import ESMDataSource
+from .utils import MinimalExploder
 
 
 class esm_datastore(Catalog):
@@ -125,6 +128,7 @@ def __init__(
         self.derivedcat = registry or default_registry
         self._entries = {}
         self._requested_variables = []
+        self._columns_with_iterables = columns_with_iterables or []
         self.datasets = {}
         self._validate_derivedcat()
 
@@ -212,6 +216,36 @@ def df(self) -> pd.DataFrame:
         """
         return self.esmcat.df
 
+    @property
+    def interactive(self) -> None:
+        """
+        Use itables to display the catalog in an interactive table. Use polars
+        for performance ideally. Fall back to pandas if not.
+
+        We have to explode columns with iterables, otherwise javascript stringifcation
+        can cause ellipsis to be rendered directly into the interactive table,
+        losing actual data and inserting junk.
+        """
+
+        try:
+            pl_df = self.esmcat._frames.polars  # type:ignore[union-attr]
+        except AttributeError:
+            pl_df = pl.from_pandas(self.df)
+
+        exploded_df = MinimalExploder(pl_df)()
+
+        return itables.show(
+            exploded_df,
+            search={'regex': True, 'caseInsensitive': True},
+            layout={'top1': 'searchPanes'},
+            searchPanes={
+                'layout': 'columns-3',
+                'cascadePanes': True,
+                'columns': [i for i, _ in enumerate(pl_df.columns)],
+            },
+            maxBytes=0,
+        )
+
     def __len__(self) -> int:
         return len(self.keys())
 

intake_esm/utils.py

@@ -2,6 +2,9 @@
 
 import importlib
 import sys
+from collections import defaultdict
+
+import polars as pl
 
 
 def show_versions(file=sys.stdout):  # pragma: no cover
@@ -119,3 +122,87 @@ def _update(self, kwargs):
     def __exit__(self, type, value, traceback):
         """Context management."""
         self._update(self.old)
+
+
+class MinimalExploder:
+    """
+    A comprehensive class for analyzing and performing minimal explosions
+    of DataFrames with multiple list columns.
+    """
+
+    def __init__(self, df: pl.DataFrame):
+        self.df = df
+        self._list_cols: list[str] | None = None
+        self._length_patterns: dict[str, tuple[int, ...]] | None = None
+        self._explodable_groups: list[list[str]] | None = None
+
+    @property
+    def list_columns(self) -> list[str]:
+        """Get all list-type columns in the DataFrame."""
+        if self._list_cols is None:
+            self._list_cols = [col for col in self.df.columns if self.df[col].dtype == pl.List]
+        return self._list_cols
+
+    @property
+    def length_patterns(self) -> dict[str, tuple[int, ...]]:
+        """Get length patterns for all list columns.
+
+        This is stored as a dictionary containing tuples of all list lengths, ie
+        'a' : (1,3,2),
+        'b' : (2,2,2),
+
+        """
+        if self._length_patterns is None:
+            self._length_patterns = self._analyze_patterns()
+        return self._length_patterns
+
+    @property
+    def explodable_groups(self) -> list[list[str]]:
+        """Get groups of columns that can be exploded together."""
+        if self._explodable_groups is None:
+            self._explodable_groups = self._compute_groups()
+        return self._explodable_groups
+
+    def _analyze_patterns(self) -> dict[str, tuple[int, ...]]:
+        """Analyze length patterns of all list columns. Returns a value
+        rather than setting self._length_patterns to shut up mypy."""
+        _length_patterns = {}
+
+        for col in self.list_columns:
+            lengths = self.df.select(pl.col(col).list.len()).to_series().to_list()
+            _length_patterns[col] = tuple(lengths)
+
+        return _length_patterns
+
+    def _compute_groups(self):
+        """Compute explodable groups based on length patterns. Returns a value
+        rather than setting self._explodable_groups to shut up mypy."""
+        pattern_groups = defaultdict(list)
+
+        for col, pattern in self.length_patterns.items():
+            pattern_groups[pattern].append(col)
+
+        return list(pattern_groups.values())
+
+    @property
+    def summary(self) -> dict:
+        """Get a summary of the explosion analysis."""
+        return {
+            'total_columns': len(self.df.columns),
+            'list_columns': len(self.list_columns),
+            'unique_patterns': len(set(self.length_patterns.values())),
+            'explodable_groups': len(self.explodable_groups),
+            'explosion_operations_needed': len(self.explodable_groups),
+            'groups': self.explodable_groups,
+        }
+
+    def __call__(self) -> pl.DataFrame:
+        """Perform the minimal explosion."""
+        if not self.list_columns:
+            return self.df
+
+        result_df = self.df
+        for group in self.explodable_groups:
+            result_df = result_df.explode(*group)
+
+        return result_df

requirements.txt

@@ -2,6 +2,7 @@ dask[complete]>=2024.12
 fastprogress>=1.0.0
 fsspec>=2024.12
 intake>=2.0.0
+itables
 netCDF4>=1.5.5
 pandas>=2.1.0
 polars>=1.24.0