don't include files list in fo and remove old stuff

Author: lhoestq

src/datasets/builder.py

@@ -24,7 +24,7 @@
 import shutil
 import time
 import urllib
-from collections.abc import Iterator, Mapping
+from collections.abc import Iterator
 from dataclasses import dataclass
 from functools import partial
 from pathlib import Path
@@ -56,7 +56,7 @@
     rename,
 )
 from .fingerprint import Hasher
-from .info import DatasetInfo, PostProcessedInfo
+from .info import DatasetInfo
 from .iterable_dataset import ArrowExamplesIterable, ExamplesIterable, IterableDataset
 from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH, camelcase_to_snakecase
 from .splits import Split, SplitDict, SplitGenerator, SplitInfo
@@ -66,7 +66,7 @@
 from .utils import tqdm as hf_tqdm
 from .utils._filelock import FileLock
 from .utils.file_utils import is_remote_url
-from .utils.info_utils import VerificationMode, get_size_checksum_dict, verify_checksums, verify_splits
+from .utils.info_utils import VerificationMode, verify_checksums, verify_splits
 from .utils.py_utils import (
     classproperty,
     convert_file_size_to_int,
@@ -406,7 +406,7 @@ def __init__(
         self.dl_manager = None
 
         # Set to True by "datasets-cli test" to generate file checksums for (deprecated) dataset_infos.json independently of verification_mode value.
-        self._record_infos = False
+        self._record_checksums = False
 
         # Set in `.download_and_prepare` once the format of the generated dataset is known
         self._file_format = None
@@ -805,7 +805,7 @@ def download_and_prepare(
                 download_config=download_config,
                 data_dir=self.config.data_dir,
                 base_path=base_path,
-                record_checksums=(self._record_infos or verification_mode == VerificationMode.ALL_CHECKS),
+                record_checksums=self._record_checksums,
             )
 
         is_local = not is_remote_filesystem(self._fs)
@@ -826,7 +826,6 @@ def download_and_prepare(
                 # We need to update the info in case some splits were added in the meantime
                 # for example when calling load_dataset from multiple workers.
                 self.info = self._load_info()
-                self.download_post_processing_resources(dl_manager)
                 return
 
             logger.info(f"Generating dataset {self.dataset_name} ({self._output_dir})")
@@ -835,7 +834,7 @@ def download_and_prepare(
                     self.info.size_in_bytes or 0, directory=Path(self._output_dir).parent
                 ):
                     raise OSError(
-                        f"Not enough disk space. Needed: {size_str(self.info.size_in_bytes or 0)} (download: {size_str(self.info.download_size or 0)}, generated: {size_str(self.info.dataset_size or 0)}, post-processed: {size_str(self.info.post_processing_size or 0)})"
+                        f"Not enough disk space. Needed: {size_str(self.info.size_in_bytes or 0)} (download: {size_str(self.info.download_size or 0)}, generated: {size_str(self.info.dataset_size or 0)}"
                     )
 
             @contextlib.contextmanager
@@ -864,7 +863,6 @@ def incomplete_dir(dirname):
                 logger.info(
                     f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} "
                     f"(download: {size_str(self.info.download_size)}, generated: {size_str(self.info.dataset_size)}, "
-                    f"post-processed: {size_str(self.info.post_processing_size)}, "
                     f"total: {size_str(self.info.size_in_bytes)}) to {self._output_dir}..."
                 )
             else:
@@ -889,15 +887,13 @@ def incomplete_dir(dirname):
                     )
                     # Sync info
                     self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())
-                    self.info.download_checksums = dl_manager.get_recorded_sizes_checksums()
+                    if dl_manager.record_checksums:
+                        self.info.download_checksums = dl_manager.get_recorded_sizes_checksums()
                     if self.info.download_size is not None:
                         self.info.size_in_bytes = self.info.dataset_size + self.info.download_size
                     # Save info
                     self._save_info()
 
-            # Download post processing resources
-            self.download_post_processing_resources(dl_manager)
-
             logger.info(
                 f"Dataset {self.dataset_name} downloaded and prepared to {self._output_dir}. "
                 f"Subsequent calls will reuse this data."
@@ -956,22 +952,6 @@ def _download_and_prepare(self, dl_manager, verification_mode, **prepare_split_k
         self.info.splits = split_dict
         self.info.download_size = dl_manager.downloaded_size
 
-    def download_post_processing_resources(self, dl_manager):
-        for split in self.info.splits or []:
-            for resource_name, resource_file_name in self._post_processing_resources(split).items():
-                if not not is_remote_filesystem(self._fs):
-                    raise NotImplementedError(f"Post processing is not supported on filesystem {self._fs}")
-                if os.sep in resource_file_name:
-                    raise ValueError(f"Resources shouldn't be in a sub-directory: {resource_file_name}")
-                resource_path = os.path.join(self._output_dir, resource_file_name)
-                if not os.path.exists(resource_path):
-                    downloaded_resource_path = self._download_post_processing_resources(
-                        split, resource_name, dl_manager
-                    )
-                    if downloaded_resource_path:
-                        logger.info(f"Downloaded post-processing resource {resource_name} as {resource_file_name}")
-                        shutil.move(downloaded_resource_path, resource_path)
-
     def _load_info(self) -> DatasetInfo:
         return DatasetInfo.from_directory(self._output_dir, storage_options=self._fs.storage_options)
 
@@ -992,18 +972,13 @@ def _make_split_generators_kwargs(self, prepare_split_kwargs):
     def as_dataset(
         self,
         split: Optional[Union[str, Split, list[str], list[Split]]] = None,
-        run_post_process=True,
-        verification_mode: Optional[Union[VerificationMode, str]] = None,
         in_memory=False,
     ) -> Union[Dataset, DatasetDict]:
         """Return a Dataset for the specified split.
 
         Args:
             split (`datasets.Split`):
                 Which subset of the data to return.
-            run_post_process (`bool`, defaults to `True`):
-                Whether to run post-processing dataset transforms and/or add
-                indexes.
             verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
                 Verification mode determining the checks to run on the
                 downloaded/processed dataset information (checksums/size/splits/...).
@@ -1046,14 +1021,10 @@ def as_dataset(
         if split is None:
             split = {s: s for s in self.info.splits}
 
-        verification_mode = VerificationMode(verification_mode or VerificationMode.BASIC_CHECKS)
-
         # Create a dataset for each of the given splits
         datasets = map_nested(
             partial(
                 self._build_single_dataset,
-                run_post_process=run_post_process,
-                verification_mode=verification_mode,
                 in_memory=in_memory,
             ),
             split,
@@ -1067,8 +1038,6 @@ def as_dataset(
     def _build_single_dataset(
         self,
         split: Union[str, ReadInstruction, Split],
-        run_post_process: bool,
-        verification_mode: VerificationMode,
         in_memory: bool = False,
     ):
         """as_dataset for a single split."""
@@ -1083,54 +1052,6 @@ def _build_single_dataset(
             split=split,
             in_memory=in_memory,
         )
-        if run_post_process:
-            for resource_file_name in self._post_processing_resources(split).values():
-                if os.sep in resource_file_name:
-                    raise ValueError(f"Resources shouldn't be in a sub-directory: {resource_file_name}")
-            resources_paths = {
-                resource_name: os.path.join(self._output_dir, resource_file_name)
-                for resource_name, resource_file_name in self._post_processing_resources(split).items()
-            }
-            post_processed = self._post_process(ds, resources_paths)
-            if post_processed is not None:
-                ds = post_processed
-                recorded_checksums = {}
-                record_checksums = False
-                for resource_name, resource_path in resources_paths.items():
-                    size_checksum = get_size_checksum_dict(resource_path)
-                    recorded_checksums[resource_name] = size_checksum
-                if verification_mode == VerificationMode.ALL_CHECKS and record_checksums:
-                    if self.info.post_processed is None or self.info.post_processed.resources_checksums is None:
-                        expected_checksums = None
-                    else:
-                        expected_checksums = self.info.post_processed.resources_checksums.get(split)
-                    verify_checksums(expected_checksums, recorded_checksums, "post processing resources")
-                if self.info.post_processed is None:
-                    self.info.post_processed = PostProcessedInfo()
-                if self.info.post_processed.resources_checksums is None:
-                    self.info.post_processed.resources_checksums = {}
-                self.info.post_processed.resources_checksums[str(split)] = recorded_checksums
-                self.info.post_processing_size = sum(
-                    checksums_dict["num_bytes"]
-                    for split_checksums_dicts in self.info.post_processed.resources_checksums.values()
-                    for checksums_dict in split_checksums_dicts.values()
-                )
-                if self.info.dataset_size is not None and self.info.download_size is not None:
-                    self.info.size_in_bytes = (
-                        self.info.dataset_size + self.info.download_size + self.info.post_processing_size
-                    )
-                self._save_info()
-                ds._info.post_processed = self.info.post_processed
-                ds._info.post_processing_size = self.info.post_processing_size
-                ds._info.size_in_bytes = self.info.size_in_bytes
-                if self.info.post_processed.features is not None:
-                    if self.info.post_processed.features.type != ds.features.type:
-                        raise ValueError(
-                            f"Post-processed features info don't match the dataset:\nGot\n{self.info.post_processed.features}\nbut expected something like\n{ds.features}"
-                        )
-                    else:
-                        ds.info.features = self.info.post_processed.features
-
         return ds
 
     def _as_dataset(self, split: Union[ReadInstruction, Split] = Split.TRAIN, in_memory: bool = False) -> Dataset:
@@ -1216,20 +1137,6 @@ def _as_streaming_dataset_single(
             ex_iterable, info=self.info, split=splits_generator.name, token_per_repo_id=token_per_repo_id
         )
 
-    def _post_process(self, dataset: Dataset, resources_paths: Mapping[str, str]) -> Optional[Dataset]:
-        """Run dataset transforms or add indexes"""
-        return None
-
-    def _post_processing_resources(self, split: str) -> dict[str, str]:
-        """Mapping resource_name -> resource_file_name"""
-        return {}
-
-    def _download_post_processing_resources(
-        self, split: str, resource_name: str, dl_manager: DownloadManager
-    ) -> Optional[str]:
-        """Download the resource using the download manager and return the downloaded path."""
-        return None
-
     @abc.abstractmethod
     def _split_generators(self, dl_manager: Union[DownloadManager, StreamingDownloadManager]):
         """Specify feature dictionary generators and dataset splits.

src/datasets/commands/test.py

@@ -144,7 +144,7 @@ def get_builders() -> Generator[DatasetBuilder, None, None]:
 
         for j, builder in enumerate(get_builders()):
             print(f"Testing builder '{builder.config.name}' ({j + 1}/{n_builders})")
-            builder._record_infos = os.path.exists(
+            builder._record_checksums = os.path.exists(
                 os.path.join(builder.get_imported_module_dir(), datasets.config.DATASETDICT_INFOS_FILENAME)
             )  # record checksums only if we need to update a (deprecated) dataset_infos.json
             builder.download_and_prepare(

src/datasets/download/download_manager.py

@@ -77,7 +77,7 @@ def __init__(
         data_dir: Optional[str] = None,
         download_config: Optional[DownloadConfig] = None,
         base_path: Optional[str] = None,
-        record_checksums=True,
+        record_checksums=False,
     ):
         """Download manager constructor.
 
@@ -93,7 +93,7 @@ def __init__(
             base_path (`str`):
                 base path that is used when relative paths are used to
                 download files. This can be a remote url.
-            record_checksums (`bool`, defaults to `True`):
+            record_checksums (`bool`, defaults to `False`):
                 Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder.
         """
         self._dataset_name = dataset_name

src/datasets/info.py

@@ -109,7 +109,7 @@ class DatasetInfo:
         features ([`Features`], *optional*):
             The features used to specify the dataset's column types.
         post_processed (`PostProcessedInfo`, *optional*):
-            Information regarding the resources of a possible post-processing of a dataset. For example, it can contain the information of an index.
+            Deprecated. Information regarding the resources of a possible post-processing of a dataset. For example, it can contain the information of an index.
         supervised_keys (`SupervisedKeysData`, *optional*):
             Specifies the input feature and the label for supervised learning if applicable for the dataset (legacy from TFDS).
         builder_name (`str`, *optional*):
@@ -125,7 +125,7 @@ class DatasetInfo:
         download_size (`int`, *optional*):
             The size of the files to download to generate the dataset, in bytes.
         post_processing_size (`int`, *optional*):
-            Size of the dataset in bytes after post-processing, if any.
+            Deprecated. Size of the dataset in bytes after post-processing, if any.
         dataset_size (`int`, *optional*):
             The combined size in bytes of the Arrow tables for all splits.
         size_in_bytes (`int`, *optional*):
@@ -140,7 +140,7 @@ class DatasetInfo:
     homepage: str = dataclasses.field(default_factory=str)
     license: str = dataclasses.field(default_factory=str)
     features: Optional[Features] = None
-    post_processed: Optional[PostProcessedInfo] = None
+    post_processed: Optional[PostProcessedInfo] = None  # kept for bawkard compat
     supervised_keys: Optional[SupervisedKeysData] = None
 
     # Set later by the builder
@@ -320,6 +320,16 @@ def _from_yaml_dict(cls, yaml_data: dict) -> "DatasetInfo":
         field_names = {f.name for f in dataclasses.fields(cls)}
         return cls(**{k: v for k, v in yaml_data.items() if k in field_names})
 
+    def __repr__(self):
+        return (
+            self.__class__.__qualname__
+            + "("
+            + ", ".join(
+                [f"{f.name}={repr(getattr(self, f.name))}" for f in dataclasses.fields(self) if getattr(self, f.name)]
+            )
+            + ")"
+        )
+
 
 class DatasetInfosDict(dict[str, DatasetInfo]):
     def write_to_directory(self, dataset_infos_dir, overwrite=False, pretty_print=False) -> None:

src/datasets/io/csv.py

@@ -60,9 +60,7 @@ def read(self):
                 base_path=base_path,
                 num_proc=self.num_proc,
             )
-            dataset = self.builder.as_dataset(
-                split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
-            )
+            dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
         return dataset
 
 

src/datasets/io/generator.py

@@ -56,9 +56,7 @@ def read(self):
                 base_path=base_path,
                 num_proc=self.num_proc,
             )
-            dataset = self.builder.as_dataset(
-                split=self.builder.config.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
-            )
+            dataset = self.builder.as_dataset(split=self.builder.config.split, in_memory=self.keep_in_memory)
             if self.fingerprint:
                 dataset._fingerprint = self.fingerprint
         return dataset

src/datasets/io/json.py

@@ -63,9 +63,7 @@ def read(self):
                 base_path=base_path,
                 num_proc=self.num_proc,
             )
-            dataset = self.builder.as_dataset(
-                split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
-            )
+            dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
         return dataset
 
 

src/datasets/io/parquet.py

@@ -66,9 +66,7 @@ def read(self):
                 base_path=base_path,
                 num_proc=self.num_proc,
             )
-            dataset = self.builder.as_dataset(
-                split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
-            )
+            dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
         return dataset
 
 

src/datasets/io/sql.py

@@ -47,9 +47,7 @@ def read(self):
         )
 
         # Build dataset for splits
-        dataset = self.builder.as_dataset(
-            split="train", verification_mode=verification_mode, in_memory=self.keep_in_memory
-        )
+        dataset = self.builder.as_dataset(split="train", in_memory=self.keep_in_memory)
         return dataset
 
 

src/datasets/io/text.py

@@ -54,7 +54,5 @@ def read(self):
                 base_path=base_path,
                 num_proc=self.num_proc,
             )
-            dataset = self.builder.as_dataset(
-                split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory
-            )
+            dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
         return dataset