From 1547abb2eeb56da5d29920349611c71adae370b8 Mon Sep 17 00:00:00 2001 From: rishab Date: Tue, 21 Jan 2025 22:19:30 +0530 Subject: [PATCH 1/5] added item_count --- jupyter_server/services/contents/filemanager.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jupyter_server/services/contents/filemanager.py b/jupyter_server/services/contents/filemanager.py index 96029d96d6..c9089699bc 100644 --- a/jupyter_server/services/contents/filemanager.py +++ b/jupyter_server/services/contents/filemanager.py @@ -293,9 +293,12 @@ def _dir_model(self, path, content=True): model = self._base_model(path) model["type"] = "directory" model["size"] = None + model["item_count"] = None if content: model["content"] = contents = [] os_dir = self._get_os_path(path) + dir_contents = os.listdir(os_dir) + model["item_count"] = len(dir_contents) for name in os.listdir(os_dir): try: os_path = os.path.join(os_dir, name) @@ -334,7 +337,6 @@ def _dir_model(self, path, content=True): os_path, exc_info=True, ) - model["format"] = "json" return model @@ -470,6 +472,7 @@ def _save_directory(self, os_path, model, path=""): if not os.path.exists(os_path): with self.perm_to_403(): os.mkdir(os_path) + model["item_count"] = 0 elif not os.path.isdir(os_path): raise web.HTTPError(400, "Not a directory: %s" % (os_path)) else: @@ -765,10 +768,12 @@ async def _dir_model(self, path, content=True): model = self._base_model(path) model["type"] = "directory" model["size"] = None + model["item_count"] = None if content: model["content"] = contents = [] os_dir = self._get_os_path(path) dir_contents = await run_sync(os.listdir, os_dir) + model["item_count"] = len(dir_contents) for name in dir_contents: try: os_path = os.path.join(os_dir, name) From 7a96565f80fa8bca11d3b9e52bf6df86406059bb Mon Sep 17 00:00:00 2001 From: rishab Date: Fri, 24 Jan 2025 21:07:12 +0530 Subject: [PATCH 2/5] added tests, doc and other minor changes --- docs/source/developers/contents.rst | 5 ++++ .../services/contents/filemanager.py | 29 ++++++++++++++----- tests/services/contents/test_manager.py | 5 ++++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/docs/source/developers/contents.rst b/docs/source/developers/contents.rst index 6910535f30..6c10b84bf2 100644 --- a/docs/source/developers/contents.rst +++ b/docs/source/developers/contents.rst @@ -59,6 +59,10 @@ Models may contain the following entries: | | ``None`` | if any. (:ref:`See | | | | Below`) | +--------------------+------------+-------------------------------+ +| **item_count** | int or | The number of items in a | +| | ``None`` | directory, or ``None`` for | +| | | files and notebooks. | ++--------------------+------------+-------------------------------+ | **format** | unicode or | The format of ``content``, | | | ``None`` | if any. (:ref:`See | | | | Below`) | @@ -110,6 +114,7 @@ model. There are three model types: **notebook**, **file**, and **directory**. - The ``content`` field contains a list of :ref:`content-free` models representing the entities in the directory. - The ``hash`` field is always ``None``. + - The ``item_count`` field contains the number of items in the directory .. note:: diff --git a/jupyter_server/services/contents/filemanager.py b/jupyter_server/services/contents/filemanager.py index c9089699bc..df9800db77 100644 --- a/jupyter_server/services/contents/filemanager.py +++ b/jupyter_server/services/contents/filemanager.py @@ -272,6 +272,7 @@ def _base_model(self, path): model["writable"] = self.is_writable(path) model["hash"] = None model["hash_algorithm"] = None + model["item_count"] = None return model @@ -293,12 +294,18 @@ def _dir_model(self, path, content=True): model = self._base_model(path) model["type"] = "directory" model["size"] = None - model["item_count"] = None + os_dir = self._get_os_path(path) + dir_contents = os.listdir(os_dir) + model["item_count"] = len( + [ + name + for name in dir_contents + if self.should_list(name) + and (self.allow_hidden or not is_file_hidden(os.path.join(os_dir, name))) + ] + ) if content: model["content"] = contents = [] - os_dir = self._get_os_path(path) - dir_contents = os.listdir(os_dir) - model["item_count"] = len(dir_contents) for name in os.listdir(os_dir): try: os_path = os.path.join(os_dir, name) @@ -768,12 +775,18 @@ async def _dir_model(self, path, content=True): model = self._base_model(path) model["type"] = "directory" model["size"] = None - model["item_count"] = None + os_dir = self._get_os_path(path) + dir_contents = await run_sync(os.listdir, os_dir) + model["item_count"] = len( + [ + name + for name in dir_contents + if self.should_list(name) + and (self.allow_hidden or not is_file_hidden(os.path.join(os_dir, name))) + ] + ) if content: model["content"] = contents = [] - os_dir = self._get_os_path(path) - dir_contents = await run_sync(os.listdir, os_dir) - model["item_count"] = len(dir_contents) for name in dir_contents: try: os_path = os.path.join(os_dir, name) diff --git a/tests/services/contents/test_manager.py b/tests/services/contents/test_manager.py index 5c3e2eca50..157038764f 100644 --- a/tests/services/contents/test_manager.py +++ b/tests/services/contents/test_manager.py @@ -590,6 +590,7 @@ async def test_get(jp_contents_manager): assert isinstance(model2, dict) assert "name" in model2 assert "path" in model2 + assert "item_count" in model2 assert "content" in model2 assert model2["name"] == "Untitled.ipynb" assert model2["path"] == "{}/{}".format(sub_dir.strip("/"), name) @@ -631,6 +632,7 @@ async def test_get(jp_contents_manager): for key, value in expected_model.items(): assert file_model[key] == value assert "created" in file_model + assert "item_count" in file_model assert "last_modified" in file_model assert file_model["hash"] @@ -639,8 +641,10 @@ async def test_get(jp_contents_manager): _make_dir(cm, "foo/bar") dirmodel = await ensure_async(cm.get("foo")) assert dirmodel["type"] == "directory" + assert "item_count" in dirmodel assert isinstance(dirmodel["content"], list) assert len(dirmodel["content"]) == 3 + assert dirmodel["item_count"] == 3 assert dirmodel["path"] == "foo" assert dirmodel["name"] == "foo" @@ -649,6 +653,7 @@ async def test_get(jp_contents_manager): model2_no_content = await ensure_async(cm.get(sub_dir + name, content=False)) file_model_no_content = await ensure_async(cm.get("foo/untitled.txt", content=False)) sub_sub_dir_no_content = await ensure_async(cm.get("foo/bar", content=False)) + assert "item_count" in model2_no_content assert sub_sub_dir_no_content["path"] == "foo/bar" assert sub_sub_dir_no_content["name"] == "bar" From 86df6a497b87fff982265933db6dd6759eafab57 Mon Sep 17 00:00:00 2001 From: rishab Date: Fri, 24 Jan 2025 21:38:30 +0530 Subject: [PATCH 3/5] minor changes --- .../services/contents/filemanager.py | 18 ++---------------- tests/services/contents/test_manager.py | 2 +- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/jupyter_server/services/contents/filemanager.py b/jupyter_server/services/contents/filemanager.py index df9800db77..d9c5000f97 100644 --- a/jupyter_server/services/contents/filemanager.py +++ b/jupyter_server/services/contents/filemanager.py @@ -296,14 +296,7 @@ def _dir_model(self, path, content=True): model["size"] = None os_dir = self._get_os_path(path) dir_contents = os.listdir(os_dir) - model["item_count"] = len( - [ - name - for name in dir_contents - if self.should_list(name) - and (self.allow_hidden or not is_file_hidden(os.path.join(os_dir, name))) - ] - ) + model["item_count"] = len(dir_contents) if content: model["content"] = contents = [] for name in os.listdir(os_dir): @@ -777,14 +770,7 @@ async def _dir_model(self, path, content=True): model["size"] = None os_dir = self._get_os_path(path) dir_contents = await run_sync(os.listdir, os_dir) - model["item_count"] = len( - [ - name - for name in dir_contents - if self.should_list(name) - and (self.allow_hidden or not is_file_hidden(os.path.join(os_dir, name))) - ] - ) + model["item_count"] = len(dir_contents) if content: model["content"] = contents = [] for name in dir_contents: diff --git a/tests/services/contents/test_manager.py b/tests/services/contents/test_manager.py index 157038764f..2972e34dba 100644 --- a/tests/services/contents/test_manager.py +++ b/tests/services/contents/test_manager.py @@ -644,7 +644,7 @@ async def test_get(jp_contents_manager): assert "item_count" in dirmodel assert isinstance(dirmodel["content"], list) assert len(dirmodel["content"]) == 3 - assert dirmodel["item_count"] == 3 + assert dirmodel["item_count"] == 4 assert dirmodel["path"] == "foo" assert dirmodel["name"] == "foo" From ca47af118b28f3b21595901b31720ddce1de5f26 Mon Sep 17 00:00:00 2001 From: rishab Date: Fri, 24 Jan 2025 22:02:02 +0530 Subject: [PATCH 4/5] added filters on counting --- .../services/contents/filemanager.py | 22 +++++++++++++++++-- tests/services/contents/test_manager.py | 2 +- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/jupyter_server/services/contents/filemanager.py b/jupyter_server/services/contents/filemanager.py index d9c5000f97..21452f95e0 100644 --- a/jupyter_server/services/contents/filemanager.py +++ b/jupyter_server/services/contents/filemanager.py @@ -296,7 +296,16 @@ def _dir_model(self, path, content=True): model["size"] = None os_dir = self._get_os_path(path) dir_contents = os.listdir(os_dir) - model["item_count"] = len(dir_contents) + filtered_count = 0 + for name in dir_contents: + try: + os_path = os.path.join(os_dir, name) + if self.should_list(name) and (self.allow_hidden or not is_file_hidden(os_path)): + filtered_count += 1 + except OSError as e: + self.log.warning("Error accessing %s: %s", os.path.join(os_dir, name), e) + + model["item_count"] = filtered_count if content: model["content"] = contents = [] for name in os.listdir(os_dir): @@ -770,7 +779,16 @@ async def _dir_model(self, path, content=True): model["size"] = None os_dir = self._get_os_path(path) dir_contents = await run_sync(os.listdir, os_dir) - model["item_count"] = len(dir_contents) + filtered_count = 0 + for name in dir_contents: + try: + os_path = os.path.join(os_dir, name) + if self.should_list(name) and (self.allow_hidden or not is_file_hidden(os_path)): + filtered_count += 1 + except OSError as e: + self.log.warning("Error accessing %s: %s", os.path.join(os_dir, name), e) + + model["item_count"] = filtered_count if content: model["content"] = contents = [] for name in dir_contents: diff --git a/tests/services/contents/test_manager.py b/tests/services/contents/test_manager.py index 2972e34dba..157038764f 100644 --- a/tests/services/contents/test_manager.py +++ b/tests/services/contents/test_manager.py @@ -644,7 +644,7 @@ async def test_get(jp_contents_manager): assert "item_count" in dirmodel assert isinstance(dirmodel["content"], list) assert len(dirmodel["content"]) == 3 - assert dirmodel["item_count"] == 4 + assert dirmodel["item_count"] == 3 assert dirmodel["path"] == "foo" assert dirmodel["name"] == "foo" From e62569cabeda870f95ffa67c71e237227be89808 Mon Sep 17 00:00:00 2001 From: Rishab87 Date: Fri, 18 Jul 2025 13:00:00 +0530 Subject: [PATCH 5/5] made it optional & updated docs --- docs/source/developers/contents.rst | 8 ++- .../services/contents/filemanager.py | 52 ++++++++++++------- tests/services/contents/test_manager.py | 1 + 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/docs/source/developers/contents.rst b/docs/source/developers/contents.rst index 6c10b84bf2..fcee40ead4 100644 --- a/docs/source/developers/contents.rst +++ b/docs/source/developers/contents.rst @@ -61,7 +61,11 @@ Models may contain the following entries: +--------------------+------------+-------------------------------+ | **item_count** | int or | The number of items in a | | | ``None`` | directory, or ``None`` for | -| | | files and notebooks. | +| | | files and notebooks. This | +| | | field is None by default | +| | | unless | +| | | ``count_directory_items`` is | +| | | set to True. | +--------------------+------------+-------------------------------+ | **format** | unicode or | The format of ``content``, | | | ``None`` | if any. (:ref:`See | @@ -115,6 +119,8 @@ model. There are three model types: **notebook**, **file**, and **directory**. models representing the entities in the directory. - The ``hash`` field is always ``None``. - The ``item_count`` field contains the number of items in the directory + which is False by default unless ``count_directory_items`` is set to + True. .. note:: diff --git a/jupyter_server/services/contents/filemanager.py b/jupyter_server/services/contents/filemanager.py index 21452f95e0..7fed4ea16c 100644 --- a/jupyter_server/services/contents/filemanager.py +++ b/jupyter_server/services/contents/filemanager.py @@ -119,6 +119,12 @@ def _checkpoints_class_default(self): if safe. And if ``delete_to_trash`` is True, the directory won't be deleted.""", ) + count_directory_items = Bool( + False, + config=True, + help="Whether to count items in directories. Disable for better performance with large/remote directories.", + ).tag(config=True) + @default("files_handler_class") def _files_handler_class_default(self): return AuthenticatedFileHandler @@ -296,16 +302,21 @@ def _dir_model(self, path, content=True): model["size"] = None os_dir = self._get_os_path(path) dir_contents = os.listdir(os_dir) - filtered_count = 0 - for name in dir_contents: - try: - os_path = os.path.join(os_dir, name) - if self.should_list(name) and (self.allow_hidden or not is_file_hidden(os_path)): - filtered_count += 1 - except OSError as e: - self.log.warning("Error accessing %s: %s", os.path.join(os_dir, name), e) - model["item_count"] = filtered_count + if self.count_directory_items: + filtered_count = 0 + for name in dir_contents: + try: + os_path = os.path.join(os_dir, name) + if self.should_list(name) and ( + self.allow_hidden or not is_file_hidden(os_path) + ): + filtered_count += 1 + except OSError as e: + self.log.warning("Error accessing %s: %s", os.path.join(os_dir, name), e) + + model["item_count"] = filtered_count + if content: model["content"] = contents = [] for name in os.listdir(os_dir): @@ -779,16 +790,21 @@ async def _dir_model(self, path, content=True): model["size"] = None os_dir = self._get_os_path(path) dir_contents = await run_sync(os.listdir, os_dir) - filtered_count = 0 - for name in dir_contents: - try: - os_path = os.path.join(os_dir, name) - if self.should_list(name) and (self.allow_hidden or not is_file_hidden(os_path)): - filtered_count += 1 - except OSError as e: - self.log.warning("Error accessing %s: %s", os.path.join(os_dir, name), e) - model["item_count"] = filtered_count + if self.count_directory_items: + filtered_count = 0 + for name in dir_contents: + try: + os_path = os.path.join(os_dir, name) + if self.should_list(name) and ( + self.allow_hidden or not is_file_hidden(os_path) + ): + filtered_count += 1 + except OSError as e: + self.log.warning("Error accessing %s: %s", os.path.join(os_dir, name), e) + + model["item_count"] = filtered_count + if content: model["content"] = contents = [] for name in dir_contents: diff --git a/tests/services/contents/test_manager.py b/tests/services/contents/test_manager.py index 157038764f..23daf2ab20 100644 --- a/tests/services/contents/test_manager.py +++ b/tests/services/contents/test_manager.py @@ -549,6 +549,7 @@ async def test_modified_date(jp_contents_manager): async def test_get(jp_contents_manager): cm = jp_contents_manager + cm.count_directory_items = True # Create a notebook model = await ensure_async(cm.new_untitled(type="notebook")) name = model["name"]