dataset_size = estimate_dataset_size(
            Path(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"]
        )
        keep_in_memory = keep_in_memory if keep_in_memory is not None else is_small_dataset(dataset_size)
        table_cls = InMemoryTable if keep_in_memory else MemoryMappedTable

        arrow_table = concat_tables(
            thread_map(
                table_cls.from_file,
                [posixpath.join(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"]],
                tqdm_class=hf_tqdm,
                desc="Loading dataset from disk",
                # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
                disable=len(state["_data_files"]) <= 16 or None,
            )
        )

        split = state["_split"]
        split = Split(split) if split is not None else split