size_categories:100K<n<1M format:parquet modality:text library:datasets library:dask library:mlcroissant library:polars region:us
---
dataset_info:
  features:
  - name: messages
    list:
    - name: content
      dtype: string
    - name: role
      dtype: string
  - name: source
    dtype: string
  splits:
  - name: train
    num_bytes: 708820401
    num_examples: 198687
  - name: test
    num_bytes: 290460314
    num_examples: 50000
  download_size: 508464627
  dataset_size: 999280715
configs:
- config_name: default
  data_files:
  - split: train
    path: data/train-*
  - split: test
    path: data/test-*
---

```python
from datasets import load_dataset, concatenate_datasets, DatasetDict

# Helper function to check for system messages
def has_no_system_messages(example):
    return not any(msg.get('role') == 'system' for msg in example['messages'])

def standardize_format(example, source):
    return {
        "messages": example["messages"],
        "source": source
    }

# Load and filter Tulu dataset
tulu_dataset = load_dataset("allenai/tulu-3-sft-mixture", split="train")

# Filter No Robots and Flan subsets from Tulu
no_robots = tulu_dataset.filter(lambda x: x['source'] == "ai2-adapt-dev/no_robots_converted")
no_robots = no_robots.filter(has_no_system_messages)

flan = tulu_dataset.filter(lambda x: x['source'] == "ai2-adapt-dev/flan_v2_converted")
flan = flan.filter(has_no_system_messages)

print(f"No Robots dataset size: {len(no_robots)}")
print(f"Flan dataset size: {len(flan)}")

# Load and filter Ultrachat dataset
ultrachat = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft")
filtered_ultrachat = ultrachat.filter(
    lambda x: len(x['messages']) < 64 and has_no_system_messages(x)
)

# Split Ultrachat into train and test
shuffled_ultrachat = filtered_ultrachat.shuffle(seed=42)
train_size = 100000
test_size = 50000

train_ultrachat = shuffled_ultrachat.select(range(train_size))
test_ultrachat = shuffled_ultrachat.select(range(train_size, train_size + test_size))

print(f"Train Ultrachat size: {len(train_ultrachat)}")
print(f"Test Ultrachat size: {len(test_ultrachat)}")

# Standardize all datasets
no_robots_std = no_robots.map(
    lambda x: standardize_format(x, "tulu/no_robots"), 
    remove_columns=no_robots.column_names
)
flan_std = flan.map(
    lambda x: standardize_format(x, "tulu/flan"), 
    remove_columns=flan.column_names
)
train_ultrachat_std = train_ultrachat.map(
    lambda x: standardize_format(x, "ultrachat"), 
    remove_columns=train_ultrachat.column_names
)
test_ultrachat_std = test_ultrachat.map(
    lambda x: standardize_format(x, "ultrachat"), 
    remove_columns=test_ultrachat.column_names
)

# Combine datasets for training
combined_train = concatenate_datasets([
    no_robots_std, 
    flan_std, 
    train_ultrachat_std
])

print(f"Combined training dataset size: {len(combined_train)}")

# Create final dataset with train and test splits
final_dataset = DatasetDict({
    'train': combined_train,
    'test': test_ultrachat_std
})

# Push to HuggingFace Hub
final_dataset.push_to_hub("kh4dien/chat-mixture")
```