Add progress/no-progress flag to CLI

Closes #240

Author: jeromekelleher

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.1.0 2024-06-10
+
+- Add -Q/--no-progress flag to CLI
+
 # 0.0.10 2024-05-15
 - Change output format of dexplode-init and dencode-init
 - Bugfix for mac progress, and change of multiprocessing startup strategy.

bio2zarr/cli.py

@@ -58,6 +58,13 @@ def list_commands(self, ctx):
     help="Force overwriting of existing directories",
 )
 
+progress = click.option(
+    "-P /-Q",
+    "--progress/--no-progress",
+    default=True,
+    help="Show progress bars (default: show)",
+)
+
 one_based = click.option(
     "--one-based",
     is_flag=True,
@@ -190,9 +197,17 @@ def show_work_summary(work_summary, json):
 @verbose
 @column_chunk_size
 @compressor
+@progress
 @worker_processes
 def explode(
-    vcfs, icf_path, force, verbose, column_chunk_size, compressor, worker_processes
+    vcfs,
+    icf_path,
+    force,
+    verbose,
+    column_chunk_size,
+    compressor,
+    progress,
+    worker_processes,
 ):
     """
     Convert VCF(s) to intermediate columnar format
@@ -205,7 +220,7 @@ def explode(
         worker_processes=worker_processes,
         column_chunk_size=column_chunk_size,
         compressor=get_compressor(compressor),
-        show_progress=True,
+        show_progress=progress,
     )
 
 
@@ -218,6 +233,7 @@ def explode(
 @compressor
 @json
 @verbose
+@progress
 @worker_processes
 def dexplode_init(
     vcfs,
@@ -228,6 +244,7 @@ def dexplode_init(
     compressor,
     json,
     verbose,
+    progress,
     worker_processes,
 ):
     """
@@ -243,7 +260,7 @@ def dexplode_init(
         column_chunk_size=column_chunk_size,
         worker_processes=worker_processes,
         compressor=get_compressor(compressor),
-        show_progress=True,
+        show_progress=progress,
     )
     show_work_summary(work_summary, json)
 
@@ -310,6 +327,7 @@ def mkschema(icf_path):
 @samples_chunk_size
 @max_variant_chunks
 @max_memory
+@progress
 @worker_processes
 def encode(
     icf_path,
@@ -321,6 +339,7 @@ def encode(
     samples_chunk_size,
     max_variant_chunks,
     max_memory,
+    progress,
     worker_processes,
 ):
     """
@@ -337,7 +356,7 @@ def encode(
         max_variant_chunks=max_variant_chunks,
         worker_processes=worker_processes,
         max_memory=max_memory,
-        show_progress=True,
+        show_progress=progress,
     )
 
 
@@ -351,6 +370,7 @@ def encode(
 @samples_chunk_size
 @max_variant_chunks
 @json
+@progress
 @verbose
 def dencode_init(
     icf_path,
@@ -362,6 +382,7 @@ def dencode_init(
     samples_chunk_size,
     max_variant_chunks,
     json,
+    progress,
     verbose,
 ):
     """
@@ -387,7 +408,7 @@ def dencode_init(
         variants_chunk_size=variants_chunk_size,
         samples_chunk_size=samples_chunk_size,
         max_variant_chunks=max_variant_chunks,
-        show_progress=True,
+        show_progress=progress,
     )
     show_work_summary(work_summary, json)
 
@@ -413,12 +434,13 @@ def dencode_partition(zarr_path, partition, verbose, one_based):
 @click.command
 @zarr_path
 @verbose
-def dencode_finalise(zarr_path, verbose):
+@progress
+def dencode_finalise(zarr_path, verbose, progress):
     """
     Final step for distributed conversion of ICF to VCF Zarr.
     """
     setup_logging(verbose)
-    vcf2zarr.encode_finalise(zarr_path, show_progress=True)
+    vcf2zarr.encode_finalise(zarr_path, show_progress=progress)
 
 
 @click.command(name="convert")
@@ -428,6 +450,7 @@ def dencode_finalise(zarr_path, verbose):
 @variants_chunk_size
 @samples_chunk_size
 @verbose
+@progress
 @worker_processes
 def convert_vcf(
     vcfs,
@@ -436,6 +459,7 @@ def convert_vcf(
     variants_chunk_size,
     samples_chunk_size,
     verbose,
+    progress,
     worker_processes,
 ):
     """
@@ -448,7 +472,7 @@ def convert_vcf(
         zarr_path,
         variants_chunk_size=variants_chunk_size,
         samples_chunk_size=samples_chunk_size,
-        show_progress=True,
+        show_progress=progress,
         worker_processes=worker_processes,
     )
 
@@ -481,6 +505,7 @@ def vcf2zarr_main():
 @click.argument("in_path", type=click.Path())
 @click.argument("zarr_path", type=click.Path())
 @worker_processes
+@progress
 @verbose
 @variants_chunk_size
 @samples_chunk_size
@@ -489,6 +514,7 @@ def convert_plink(
     zarr_path,
     verbose,
     worker_processes,
+    progress,
     variants_chunk_size,
     samples_chunk_size,
 ):
@@ -499,7 +525,7 @@ def convert_plink(
     plink.convert(
         in_path,
         zarr_path,
-        show_progress=True,
+        show_progress=progress,
         worker_processes=worker_processes,
         samples_chunk_size=samples_chunk_size,
         variants_chunk_size=variants_chunk_size,

tests/test_cli.py

@@ -69,21 +69,22 @@ def asjson(self):
 class TestWithMocks:
     vcf_path = "tests/data/vcf/sample.vcf.gz"
 
+    @pytest.mark.parametrize(("progress", "flag"), [(True, "-P"), (False, "-Q")])
     @mock.patch("bio2zarr.vcf2zarr.explode")
-    def test_vcf_explode(self, mocked, tmp_path):
+    def test_vcf_explode(self, mocked, tmp_path, progress, flag):
         icf_path = tmp_path / "icf"
         runner = ct.CliRunner(mix_stderr=False)
         result = runner.invoke(
             cli.vcf2zarr_main,
-            f"explode {self.vcf_path} {icf_path}",
+            f"explode {self.vcf_path} {icf_path} {flag}",
             catch_exceptions=False,
         )
         assert result.exit_code == 0
         assert len(result.stdout) == 0
         assert len(result.stderr) == 0
-        mocked.assert_called_once_with(
-            str(icf_path), (self.vcf_path,), **DEFAULT_EXPLODE_ARGS
-        )
+        args = dict(DEFAULT_EXPLODE_ARGS)
+        args["show_progress"] = progress
+        mocked.assert_called_once_with(str(icf_path), (self.vcf_path,), **args)
 
     @pytest.mark.parametrize("compressor", ["lz4", "zstd"])
     @mock.patch("bio2zarr.vcf2zarr.explode")
@@ -286,23 +287,26 @@ def test_vcf_explode_missing_and_existing_vcf(self, mocked, tmp_path):
         assert "'no_such_file' does not exist" in result.stderr
         mocked.assert_not_called()
 
+    @pytest.mark.parametrize(("progress", "flag"), [(True, "-P"), (False, "-Q")])
     @mock.patch("bio2zarr.vcf2zarr.explode_init", return_value=FakeWorkSummary(5))
-    def test_vcf_dexplode_init(self, mocked, tmp_path):
+    def test_vcf_dexplode_init(self, mocked, tmp_path, progress, flag):
         runner = ct.CliRunner(mix_stderr=False)
         icf_path = tmp_path / "icf"
         result = runner.invoke(
             cli.vcf2zarr_main,
-            f"dexplode-init {self.vcf_path} {icf_path} 5",
+            f"dexplode-init {self.vcf_path} {icf_path} 5 {flag}",
             catch_exceptions=False,
         )
         assert result.exit_code == 0
         assert len(result.stderr) == 0
         assert list(result.stdout.split()) == ["num_partitions", "5"]
+        args = dict(DEFAULT_DEXPLODE_INIT_ARGS)
+        args["show_progress"] = progress
         mocked.assert_called_once_with(
             str(icf_path),
             (self.vcf_path,),
             target_num_partitions=5,
-            **DEFAULT_DEXPLODE_INIT_ARGS,
+            **args,
         )
 
     @pytest.mark.parametrize("num_partitions", ["-- -1", "0", "asdf", "1.112"])
@@ -421,43 +425,51 @@ def test_mkschema(self, mocked, tmp_path):
         # mocked.assert_called_once_with("path", stdout)
         mocked.assert_called_once()
 
+    @pytest.mark.parametrize(("progress", "flag"), [(True, "-P"), (False, "-Q")])
     @mock.patch("bio2zarr.vcf2zarr.encode")
-    def test_encode(self, mocked, tmp_path):
+    def test_encode(self, mocked, tmp_path, progress, flag):
         icf_path = tmp_path / "icf"
         icf_path.mkdir()
         zarr_path = tmp_path / "zarr"
         runner = ct.CliRunner(mix_stderr=False)
         result = runner.invoke(
-            cli.vcf2zarr_main, f"encode {icf_path} {zarr_path}", catch_exceptions=False
+            cli.vcf2zarr_main,
+            f"encode {icf_path} {zarr_path} {flag}",
+            catch_exceptions=False,
         )
         assert result.exit_code == 0
         assert len(result.stdout) == 0
         assert len(result.stderr) == 0
+        args = DEFAULT_ENCODE_ARGS
+        args["show_progress"] = progress
         mocked.assert_called_once_with(
             str(icf_path),
             str(zarr_path),
-            **DEFAULT_ENCODE_ARGS,
+            **args,
         )
 
+    @pytest.mark.parametrize(("progress", "flag"), [(True, "-P"), (False, "-Q")])
     @mock.patch("bio2zarr.vcf2zarr.encode_init", return_value=FakeWorkSummary(10))
-    def test_dencode_init(self, mocked, tmp_path):
+    def test_dencode_init(self, mocked, tmp_path, progress, flag):
         icf_path = tmp_path / "icf"
         icf_path.mkdir()
         zarr_path = tmp_path / "zarr"
         runner = ct.CliRunner(mix_stderr=False)
         result = runner.invoke(
             cli.vcf2zarr_main,
-            f"dencode-init {icf_path} {zarr_path} 10",
+            f"dencode-init {icf_path} {zarr_path} 10 {flag}",
             catch_exceptions=False,
         )
         assert result.exit_code == 0
         assert list(result.stdout.split()) == ["num_partitions", "10"]
         assert len(result.stderr) == 0
+        args = DEFAULT_DENCODE_INIT_ARGS
+        args["show_progress"] = progress
         mocked.assert_called_once_with(
             str(icf_path),
             str(zarr_path),
             target_num_partitions=10,
-            **DEFAULT_DENCODE_INIT_ARGS,
+            **args,
         )
 
     @mock.patch("bio2zarr.vcf2zarr.encode_partition")
@@ -494,32 +506,40 @@ def test_vcf_dencode_partition_one_based(self, mocked, tmp_path):
             str(zarr_path), 0, **DEFAULT_DENCODE_PARTITION_ARGS
         )
 
+    @pytest.mark.parametrize(("progress", "flag"), [(True, "-P"), (False, "-Q")])
     @mock.patch("bio2zarr.vcf2zarr.encode_finalise")
-    def test_vcf_dencode_finalise(self, mocked, tmp_path):
+    def test_vcf_dencode_finalise(self, mocked, tmp_path, progress, flag):
         runner = ct.CliRunner(mix_stderr=False)
         result = runner.invoke(
-            cli.vcf2zarr_main, f"dencode-finalise {tmp_path}", catch_exceptions=False
+            cli.vcf2zarr_main,
+            f"dencode-finalise {tmp_path} {flag}",
+            catch_exceptions=False,
         )
         assert result.exit_code == 0
         assert len(result.stdout) == 0
         assert len(result.stderr) == 0
-        mocked.assert_called_once_with(str(tmp_path), **DEFAULT_DENCODE_FINALISE_ARGS)
+        args = DEFAULT_DENCODE_FINALISE_ARGS
+        args["show_progress"] = progress
+        mocked.assert_called_once_with(str(tmp_path), **args)
 
+    @pytest.mark.parametrize(("progress", "flag"), [(True, "-P"), (False, "-Q")])
     @mock.patch("bio2zarr.vcf2zarr.convert")
-    def test_convert_vcf(self, mocked):
+    def test_convert_vcf(self, mocked, progress, flag):
         runner = ct.CliRunner(mix_stderr=False)
         result = runner.invoke(
             cli.vcf2zarr_main,
-            f"convert {self.vcf_path} zarr_path",
+            f"convert {self.vcf_path} zarr_path {flag}",
             catch_exceptions=False,
         )
         assert result.exit_code == 0
         assert len(result.stdout) == 0
         assert len(result.stderr) == 0
+        args = dict(DEFAULT_CONVERT_ARGS)
+        args["show_progress"] = progress
         mocked.assert_called_once_with(
             (self.vcf_path,),
             "zarr_path",
-            **DEFAULT_CONVERT_ARGS,
+            **args,
         )
 
     @pytest.mark.parametrize("response", ["n", "N", "No"])
@@ -538,16 +558,19 @@ def test_vcf_convert_overwrite_zarr_confirm_no(self, mocked, tmp_path, response)
         assert "Aborted" in result.stderr
         mocked.assert_not_called()
 
+    @pytest.mark.parametrize(("progress", "flag"), [(True, "-P"), (False, "-Q")])
     @mock.patch("bio2zarr.plink.convert")
-    def test_convert_plink(self, mocked):
+    def test_convert_plink(self, mocked, progress, flag):
         runner = ct.CliRunner(mix_stderr=False)
         result = runner.invoke(
-            cli.plink2zarr, ["convert", "in", "out"], catch_exceptions=False
+            cli.plink2zarr, ["convert", "in", "out", flag], catch_exceptions=False
         )
         assert result.exit_code == 0
         assert len(result.stdout) == 0
         assert len(result.stderr) == 0
-        mocked.assert_called_once_with("in", "out", **DEFAULT_CONVERT_ARGS)
+        args = dict(DEFAULT_CONVERT_ARGS)
+        args["show_progress"] = progress
+        mocked.assert_called_once_with("in", "out", **args)
 
     @pytest.mark.parametrize("response", ["y", "Y", "yes"])
     @mock.patch("bio2zarr.vcf2zarr.convert")
@@ -578,10 +601,11 @@ def test_dexplode(self, tmp_path, one_based):
         runner = ct.CliRunner(mix_stderr=False)
         result = runner.invoke(
             cli.vcf2zarr_main,
-            f"dexplode-init {self.vcf_path} {icf_path} 5 --json",
+            f"dexplode-init {self.vcf_path} {icf_path} 5 --json -Q",
             catch_exceptions=False,
         )
         assert result.exit_code == 0
+        assert len(result.stderr) == 0
         assert json.loads(result.stdout)["num_partitions"] == 3
 
         for j in range(3):
@@ -595,6 +619,7 @@ def test_dexplode(self, tmp_path, one_based):
             cli.vcf2zarr_main, f"dexplode-finalise {icf_path}", catch_exceptions=False
         )
         assert result.exit_code == 0
+        assert len(result.stderr) == 0
 
         result = runner.invoke(
             cli.vcf2zarr_main, f"inspect {icf_path}", catch_exceptions=False