size_categories:1M<n<10M format:parquet modality:text library:datasets library:dask library:mlcroissant library:polars region:us
---
dataset_info:
  features:
  - name: conversations
    list:
    - name: content
      dtype: string
    - name: role
      dtype: string
  - name: source
    dtype: string
  - name: conversation_id
    dtype: int64
  splits:
  - name: train
    num_bytes: 3232157279.4142313
    num_examples: 1724908
  download_size: 1512890847
  dataset_size: 3232157279.4142313
configs:
- config_name: default
  data_files:
  - split: train
    path: data/train-*
---
```
from datasets import load_dataset
ds = load_dataset("pxyyy/RLHFlow_mixture", split='train')

def filter_data(example):
    messages = example['messages']
    if len(messages) < 2:
        return False
    for i in range(len(messages)):
        if messages[0]['role'] == 'system':
            if i == 0:
                turn = 'system'
            elif i % 2 == 1:
                turn = 'user'
            else:
                turn = 'assistant'
        else:
            if i % 2 == 0:
                turn = 'user'
            else:
                turn = 'assistant'

        if messages[i]['role'] != turn:
            return False
        if len(messages[i]['content']) == 0:
            return False
    return True

cnt = 0
for sample in ds:
    all_roles = [mes['role'] for mes in sample['messages']]
    if 'system' in all_roles:
        cnt += 1
print(cnt)

new_ds = ds.filter(filter_data, num_proc=8)
new_ds = new_ds.shuffle(seed=42)
new_ds = new_ds.rename_column("messages", "conversations")
new_ds.push_to_hub('pxyyy/RLHFlow_mixture_clean_empty_round')
```