@aulemahal suggestion (edge case, single item iterable column)

Author: charles-turner-1

intake_esm/cat.py

@@ -516,7 +516,7 @@ def pandas(self) -> pd.DataFrame:
 
         if self.pl_df is not None:
             self.df = self.pl_df.to_pandas(use_pyarrow_extension_array=True)
-            for colname in self.columns_with_iterables:
+            for colname in self.columns_with_iterables:  # Can this be done in one hit?
                 self.df[colname] = self.df[colname].apply(tuple)
             return self.df
 
@@ -660,6 +660,7 @@ def _read_csv_pl(self) -> FramesModel:
                 pl.col(colname)
                 .str.replace('^.', '[')  # Replace first/last chars with [ or ].
                 .str.replace('.$', ']')  # set/tuple => list
+                .str.replace(',]$', ']')  # Remove trailing commas
                 .str.replace_all("'", '"')
                 .str.json_decode()  # This is to do with the way polars reads json - single versus double quotes
                 for colname in converters.keys()

tests/sample-catalogs/access-single-item-iterables.csv

@@ -0,0 +1,4 @@
+filename,file_id,path,filename_timestamp,frequency,start_date,end_date,variable,variable_long_name,variable_standard_name,variable_cell_methods,variable_units,realm
+iceh.1900-01.nc,iceh_XXXX_XX,/g/data/ik11/outputs/access-om2-01/01deg_jra55v13_ryf9091/output000/ice/OUTPUT/iceh.1900-01.nc,1900-01,1mon,"1900-01-01, 00:00:00","1900-02-01, 00:00:00","('time',)","('model time',)","('',)","('',)","('days since 1900-01-01 00:00:00',)",seaIce
+iceh.1900-02.nc,iceh_XXXX_XX,/g/data/ik11/outputs/access-om2-01/01deg_jra55v13_ryf9091/output000/ice/OUTPUT/iceh.1900-02.nc,1900-02,1mon,"1900-02-01, 00:00:00","1900-03-01, 00:00:00","('time',)","('model time',)","('',)","('',)","('days since 1900-01-01 00:00:00',)",seaIce
+iceh.1900-03.nc,iceh_XXXX_XX,/g/data/ik11/outputs/access-om2-01/01deg_jra55v13_ryf9091/output000/ice/OUTPUT/iceh.1900-03.nc,1900-03,1mon,"1900-03-01, 00:00:00","1900-04-01, 00:00:00","('time',)","('model time',)","('',)","('',)","('days since 1900-01-01 00:00:00',)",seaIce

tests/sample-catalogs/access-single-item-iterables.json

@@ -0,0 +1,81 @@
+{
+  "esmcat_version": "0.0.1",
+  "attributes": [
+    {
+      "column_name": "filename",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "file_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "path",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "filename_timestamp",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "frequency",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "start_date",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "end_date",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "variable",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "variable_long_name",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "variable_standard_name",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "variable_cell_methods",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "variable_units",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "realm",
+      "vocabulary": ""
+    }
+  ],
+  "assets": {
+    "column_name": "path",
+    "format": "netcdf",
+    "format_column_name": null
+  },
+  "aggregation_control": {
+    "variable_column_name": "variable",
+    "groupby_attrs": ["file_id", "frequency"],
+    "aggregations": [
+      {
+        "type": "join_existing",
+        "attribute_name": "start_date",
+        "options": {
+          "dim": "time",
+          "combine": "by_coords"
+        }
+      }
+    ]
+  },
+  "id": "01deg_jra55v13_ryf9091",
+  "description": "0.1 degree ACCESS-OM2 global model configuration with JRA55-do v1.3 RYF9091  repeat year forcing (May 1990 to Apr 1991)",
+  "title": null,
+  "last_updated": "2025-03-04T01:25:35Z",
+  "catalog_file": "access-single-item-iterables.csv"
+}

tests/test_cat.py

@@ -13,6 +13,7 @@
     access_columns_with_lists_cat,
     access_columns_with_sets_cat,
     access_columns_with_tuples_cat,
+    access_single_item_iterables_cat,
     catalog_dict_records,
     cdf_cat_sample_cesmle,
     cdf_cat_sample_cmip5,
@@ -148,6 +149,7 @@ def test_esmcatmodel_unique_and_nunique(query, expected_unique_vals, expected_nu
         (access_columns_with_lists_cat, list),
         (access_columns_with_tuples_cat, tuple),
         (access_columns_with_sets_cat, set),
+        (access_single_item_iterables_cat, tuple),
     ],
 )
 def test_esmcatmodel_roundtrip_itercols_type_stable(catalog_file, expected_type):

tests/utils.py

@@ -23,6 +23,9 @@
     here, 'sample-catalogs/access-columns-with-tuples.json'
 )
 access_columns_with_sets_cat = os.path.join(here, 'sample-catalogs/access-columns-with-sets.json')
+access_single_item_iterables_cat = os.path.join(
+    here, 'sample-catalogs/access-single-item-iterables.json'
+)
 zarr_v2_cat = os.path.join(here, 'sample-catalogs/cesm1-lens-zarr2.json')
 zarr_v3_cat = os.path.join(here, 'sample-catalogs/cesm1-lens-zarr2.json')