Remove unused data files optims (#7985)

* remove unused data files optims

* style

Author: lhoestq

src/datasets/config.py

@@ -257,9 +257,7 @@
 STREAMING_OPEN_RETRY_INTERVAL = 5
 
 # Datasets repositories exploration
-DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 200
-GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 10
-ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE = 200
+ARCHIVES_MAX_NUMBER_FOR_MODULE_INFERENCE = 10
 
 # Async map functions
 MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL = 1000

src/datasets/load.py

@@ -218,7 +218,7 @@ def infer_module_for_data_files_list(
     """
     extensions_counter = Counter(
         ("." + suffix.lower(), xbasename(filepath) in FolderBasedBuilder.METADATA_FILENAMES)
-        for filepath in data_files_list[: config.DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE]
+        for filepath in data_files_list
         for suffix in xbasename(filepath).split(".")[1:]
     )
     if extensions_counter:
@@ -255,14 +255,11 @@ def infer_module_for_data_files_list_in_archives(
     for filepath in data_files_list:
         if str(filepath).endswith(".zip"):
             archive_files_counter += 1
-            if archive_files_counter > config.GLOBBED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE:
+            if archive_files_counter > config.ARCHIVES_MAX_NUMBER_FOR_MODULE_INFERENCE:
                 break
             extracted = xjoin(StreamingDownloadManager().extract(filepath), "**")
             archived_files += [
-                f.split("::")[0]
-                for f in xglob(extracted, recursive=True, download_config=download_config)[
-                    : config.ARCHIVED_DATA_FILES_MAX_NUMBER_FOR_MODULE_INFERENCE
-                ]
+                f.split("::")[0] for f in xglob(extracted, recursive=True, download_config=download_config)
             ]
     extensions_counter = Counter(
         "." + suffix.lower() for filepath in archived_files for suffix in xbasename(filepath).split(".")[1:]

tests/test_arrow_dataset.py

@@ -1926,7 +1926,7 @@ def test_filter(self, in_memory):
             with self._create_dummy_dataset(in_memory, tmp_dir) as dset:
                 dset.set_format("numpy")
                 fingerprint = dset._fingerprint
-                with dset.filter(lambda x: (int(x["filename"][-1]) % 2 == 0)) as dset_filter_even_num:
+                with dset.filter(lambda x: int(x["filename"][-1]) % 2 == 0) as dset_filter_even_num:
                     self.assertEqual(len(dset_filter_even_num), 15)
                     self.assertDictEqual(dset.features, Features({"filename": Value("string")}))
                     self.assertDictEqual(dset_filter_even_num.features, Features({"filename": Value("string")}))