batch = batch.split("\n\n")
                        pa_table = pa.Table.from_arrays(
                            [pa.array([example for example in batch[:-1] if example])], names=pa_table_names
                        )
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                        yield (file_idx, batch_idx), self._cast_table(pa_table)
                        batch_idx += 1
                        batch = batch[-1]
                    if batch:
                        pa_table = pa.Table.from_arrays([pa.array([batch])], names=pa_table_names)
                        yield (file_idx, batch_idx), self._cast_table(pa_table)
                elif self.config.sample_by == "document":