batch = [line.rstrip("\n") for line in batch]
                        pa_table = pa.Table.from_arrays([pa.array(batch)], names=pa_table_names)
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                        yield (file_idx, batch_idx), self._cast_table(pa_table)
                        batch_idx += 1
                elif self.config.sample_by == "paragraph":
                    batch_idx = 0
                    batch = ""
                    while True:
                        new_batch = f.read(self.config.chunksize)
                        if not new_batch:
                            break
                        batch += new_batch
                        batch += f.readline()  # finish current line