Add progress bar and match formal schema

Author: DavidStirling

omero2pandas/__init__.py

@@ -242,9 +242,8 @@ def upload_table(source, table_name, parent_id=None, parent_type='Image',
         if local_path or remote_path:
             if not register_table:
                 raise ValueError("Remote table support is not installed")
-            ann_id = register_table(source, table_name, links,
-                                    connector.server, chunk_size,
-                                    local_path, remote_path)
+            ann_id = register_table(
+                source, chunk_size, local_path, remote_path)
         else:
             ann_id = create_table(source, table_name, links, conn, chunk_size)
         if ann_id is None:

omero2pandas/remote.py

@@ -10,7 +10,9 @@
 from pathlib import Path, PurePosixPath
 import time
 
+import pandas as pd
 import tiledb
+from tqdm.auto import tqdm
 
 LOGGER = logging.getLogger(__name__)
 
@@ -32,9 +34,26 @@ def register_table(source, chunk_size, local_path, remote_path):
     LOGGER.info("Writing data to TileDB")
     # Export table
     if isinstance(source, (str, Path)):
-        tiledb.from_csv(write_path, source, chunksize=chunk_size)
+        data_iterator = pd.read_csv(source, chunksize=chunk_size)
+        total_rows = None
     else:
-        tiledb.from_pandas(write_path, source, chunksize=chunk_size)
+        data_iterator = (source.iloc[i:i + chunk_size]
+                         for i in range(0, len(source), chunk_size))
+        total_rows = len(source)
+    progress_monitor = tqdm(
+        desc="Generating TileDB file...", initial=1, dynamic_ncols=True,
+        total=total_rows,
+        bar_format='{desc}: {percentage:3.0f}%|{bar}| '
+                   '{n_fmt}/{total_fmt} rows, {elapsed} {postfix}')
+    row_idx = 0
+    for chunk in data_iterator:
+        tiledb.from_pandas(write_path, chunk, sparse=True, full_domain=True,
+                           tile=10000, attr_filters=None,
+                           row_start_idx=row_idx, allows_duplicates=False,
+                           mode="append" if row_idx else "ingest")
+        progress_monitor.update(len(chunk))
+        row_idx += len(chunk)
+    progress_monitor.close()
     LOGGER.debug("Appending metadata to TileDB")
     # Append omero metadata
     with tiledb.open(write_path, mode="w") as array: