logger.debug(f"Finished processing shard number {rank} of {num_shards}.")
                                transformed_shards[rank] = content
                            else:
                                pbar.update(content)
                # Avoids PermissionError on Windows (the error: https://github.com/huggingface/datasets/actions/runs/4026734820/jobs/6921621805)
                for kwargs in kwargs_per_job:
                    del kwargs["shard"]
            else:
                logger.info(f"Loading cached processed dataset at {format_cache_file_name(cache_file_name, '*')}")
            assert None not in transformed_shards, (
                f"Failed to retrieve results from map: result list {transformed_shards} still contains None - at least one worker failed to return its results"
            )
            logger.info(f"Concatenating {num_proc} shards")
            result = _concatenate_map_style_datasets(transformed_shards)