DatasetGenerationError while loading dataset
Hello!I’m trying to download dataset verbrannter/invoice_dataset_large_cleaned_2 and get DatasetGenerationError: An error occurred while generating the dataset during the Generating train split.Can you help me?RegardsDownloading data files: 100%2/2 [00:00<00:00, 64.17it/s]Extracting data files: 100%2/2 [00:00<00:00, 21.97it/s]Generating train split: 0%0/25434 [00:00<?, ? examples/s]FileNotFoundError                         Traceback (most recent call last)File ~\anaconda3\envs\myenv_nlp\lib\site-packages\datasets\builder.py:1916, in ArrowBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, job_id)1915 try:→ 1916     writer = writer_class(1917         features=self.info.features,1918         path=fpath.replace(“SSSSS”, f"{shard_id:05d}“).replace(“JJJJJ”, f”{job_id:05d}"),1919         writer_batch_size=self._writer_batch_size,1920         storage_options=self._fs.storage_options,1921         embed_local_files=embed_local_files,1922     )1923     try:File ~\anaconda3\envs\myenv_nlp\lib\site-packages\datasets\arrow_writer.py:334, in ArrowWriter.init(self, schema, features, path, stream, fingerprint, writer_batch_size, hash_salt, check_duplicates, disable_nullable, update_features, with_metadata, unit, embed_local_files, storage_options)329 self._path = (330     fs_token_paths[2][0]331     if not is_remote_filesystem(self._fs)332     else self._fs.unstrip_protocol(fs_token_paths[2][0])333 )→ 334 self.stream = self._fs.open(fs_token_paths[2][0], “wb”)335 self._closable_stream = TrueFile ~\anaconda3\envs\myenv_nlp\lib\site-packages\fsspec\spec.py:1241, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)1240 ac = kwargs.pop(“autocommit”, not self._intrans)→ 1241 f = self._open(1242     path,1243     mode=mode,1244     block_size=block_size,1245     autocommit=ac,1246     cache_options=cache_options,1247     **kwargs,1248 )1249 if compression is not None:File ~\anaconda3\envs\myenv_nlp\lib\site-packages\fsspec\implementations\local.py:184, in LocalFileSystem._open(self, path, mode, block_size, **kwargs)183     self.makedirs(self._parent(path), exist_ok=True)→ 184 return LocalFileOpener(path, mode, fs=self, **kwargs)File ~\anaconda3\envs\myenv_nlp\lib\site-packages\fsspec\implementations\local.py:315, in LocalFileOpener.init(self, path, mode, autocommit, fs, compression, **kwargs)314 self.blocksize = io.DEFAULT_BUFFER_SIZE→ 315 self._open()File ~\anaconda3\envs\myenv_nlp\lib\site-packages\fsspec\implementations\local.py:320, in LocalFileOpener._open(self)319 if self.autocommit or “w” not in self.mode:→ 320     self.f = open(self.path, mode=self.mode)321     if self.compression:FileNotFoundError: [Errno 2] No such file or directory: ‘C:/Users/xllcr/.cache/huggingface/datasets/verbrannter___invoice_dataset_large_cleaned_2/default-b6d7d315edb1e6d7/0.0.0/0111277fb19b16f696664cde7f0cb90f833dec72db2cc73cfdf87e697f78fe02.incomplete/invoice_dataset_large_cleaned_2-train-00000-00000-of-NNNNN.arrow’The above exception was the direct cause of the following exception:DatasetGenerationError                    Traceback (most recent call last)Cell In [2], line 21 from datasets import load_dataset----> 2 dataset = load_dataset(“verbrannter/invoice_dataset_large_cleaned_2”)File ~\anaconda3\envs\myenv_nlp\lib\site-packages\datasets\load.py:2153, in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, verification_mode, ignore_verifications, keep_in_memory, save_infos, revision, token, use_auth_token, task, streaming, num_proc, storage_options, **config_kwargs)2150 try_from_REDACTED_TOKEN = path not in _PACKAGED_DATASETS_MODULES2152 # Download and prepare data→ 2153 builder_instance.download_and_prepare(2154     download_config=download_config,2155     download_mode=download_mode,2156     verification_mode=verification_mode,2157     try_from_REDACTED_TOKEN=try_from_REDACTED_TOKEN,2158     num_proc=num_proc,2159     storage_options=storage_options,2160 )2162 # Build dataset for splits2163 keep_in_memory = (2164     keep_in_memory if keep_in_memory is not None else is_small_dataset(builder_instance.info.dataset_size)2165 )File ~\anaconda3\envs\myenv_nlp\lib\site-packages\datasets\builder.py:954, in DatasetBuilder.download_and_prepare(self, output_dir, download_config, download_mode, verification_mode, ignore_verifications, try_from_REDACTED_TOKEN, dl_manager, base_path, use_auth_token, file_format, max_shard_size, num_proc, storage_options, **download_and_prepare_kwargs)952     if num_proc is not None:953         prepare_split_kwargs[“num_proc”] = num_proc→ 954     self._download_and_prepare(955         dl_manager=dl_manager,956         verification_mode=verification_mode,957         **prepare_split_kwargs,958         **download_and_prepare_kwargs,959     )960 # Sync info961 self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())File ~\anaconda3\envs\myenv_nlp\lib\site-packages\datasets\builder.py:1049, in DatasetBuilder._download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs)1045 split_dict.add(split_generator.split_info)1047 try:1048     # Prepare split will record examples associated to the split→ 1049     self._prepare_split(split_generator, **prepare_split_kwargs)1050 except OSError as e:1051     raise OSError(1052         "Cannot find data file. "1053         + (self.manual_download_instructions or “”)1054         + “\nOriginal error:\n”1055         + str(e)1056     ) from NoneFile ~\anaconda3\envs\myenv_nlp\lib\site-packages\datasets\builder.py:1813, in ArrowBasedBuilder._prepare_split(self, split_generator, file_format, num_proc, max_shard_size)1811 job_id = 01812 with pbar:→ 1813     for job_id, done, content in self._prepare_split_single(1814         gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args1815     ):1816         if done:1817             result = contentFile ~\anaconda3\envs\myenv_nlp\lib\site-packages\datasets\builder.py:1958, in ArrowBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, job_id)1956     if isinstance(e, SchemaInferenceError) and e.contextis not None:1957         e = e.context→ 1958     raise DatasetGenerationError(“An error occurred while generating the dataset”) from e1960 yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths)DatasetGenerationError: An error occurred while generating the dataset
2023-10-25T11:34:50Z
I met same problem ：FileNotFoundError: [Errno 2] No such file or directory: 'C:/Users/Dell/.cache/huggingface/datasets/zeroshot___twitter-financial-news-sentiment/default-9316297f35209a5d/0.0.0/eea64c71ca8b46dd3f537ed218fc9bf495d5707789152eb2764f5c78fa66d59d.incomplete/twitter-financial-news-sentiment-validation-00000-00000-of-NNNNN.arrowI find there is no ‘default-9316297f35209a5d/0.0.0/eea64c71ca8b46dd3f537ed218fc9bf495d5707789152eb2764f5c78fa66d59d.incomplete’ directory under ‘'C:/Users/Dell/.cache/huggingface/datasets/zeroshot___twitter-financial-news-sentiment/’ directory
2023-10-26T14:30:21Z
Hmm, that’s weird. Does this error happen consistently?
2023-10-26T14:53:00Z
It works with the version 2.13.2 and not after.It works with the last version (2.14.6) with the parameter streaming=true of load_dataset.It works on Colab (2.14.6) but not with jupyter notebook.Hopes it can help.