shards = (self.shard(num_shards=num_shards, index=i, contiguous=True) for i in range(num_shards))

        if decodable_columns:
            from .io.parquet import get_writer_batch_size

            def shards_with_embedded_external_files(shards: Iterator[Dataset]) -> Iterator[Dataset]:
                for shard in shards:
                    format = shard.format
                    shard = shard.with_format("arrow")
                    shard = shard.map(
                        embed_table_storage,
                        batched=True,
                        batch_size=get_writer_batch_size(shard.features),
                        keep_in_memory=True,
                    )
                    shard = shard.with_format(**format)
                    yield shard

            shards = shards_with_embedded_external_files(shards)

        api = HfApi(endpoint=config.HF_ENDPOINT, token=token)