size_categories:100K<n<1M format:parquet modality:text library:datasets library:dask library:mlcroissant library:polars region:us
---
dataset_info:
  features:
  - name: messages
    list:
    - name: content
      dtype: string
    - name: role
      dtype: string
  - name: source
    dtype: string
  - name: from
    dtype: string
  splits:
  - name: train
    num_bytes: 586752589.0623015
    num_examples: 150000
  download_size: 254473617
  dataset_size: 586752589.0623015
configs:
- config_name: default
  data_files:
  - split: train
    path: data/train-*
---
# Dataset Card for "caphesuada-150k"

```python
"""
- The dolphin_cqa_long is not filtered well imo. 
    + It contains unwanted MQA questions.
    + It does not contain enough unanswerable questions.
- Alter the dolphin_cqa_long dataset with the new orca dataset (50K):
    v 50k ultrachat first turn (datht)
    v 10k squad_paraphrases 
    v 15k answerable CQA - dolphin
    v 25k answerable CQA - Ultrachat-AEM
    v 20k multi source CQA - WebGLM
    v 15k for CQA with no answer and long excuse - Orca
    v 15k for CQA with no answer and short excuse - Orca
    v 7.5k for Wizard
- Use new format
    + New format as zephyr
"""

from copy import deepcopy
from datasets import load_dataset
def zephyr_format(example):
    source = example["source"]
    translator = example["from"]
    messages = []
    if source in [
        "webglm", 
        "squad_paraphrases", 
        "dolphin_cqa_long",
    ]:
        system_prompt = example["instruction"]
        human_turn = example["input"]
        bot_turn = example["output"]
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": human_turn},
            {"role": "assistant", "content": bot_turn}
        ]
    elif source == "wizard_nocode":
        system_prompt = ''
        human_turn = example["instruction"]
        bot_turn = example["output"]
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": human_turn},
            {"role": "assistant", "content": bot_turn}
        ]
    elif source == "ultrachat":
        messages = example["messages"]
        messages.insert(0, {"role": "system", "content": ""})
    elif source == "ultra-aem":
        data = example["data"]
        system_prompt = "You specialize in assisting with existing materials. Please do your best."
        human_turn = data[0]
        bot_turn = data[1]
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": human_turn},
            {"role": "assistant", "content": bot_turn}
        ]   
    elif source in ["null-long", "null-short"]:
        system_prompt = example["system_prompt"]
        human_turn = example["question"]
        bot_turn = example["response"]
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": human_turn},
            {"role": "assistant", "content": bot_turn}
        ]
        
    ## Remove  
    key_names = deepcopy(list(example))
    for key in key_names:
        example.pop(key)
        
    result = {
        "messages": messages,
        "source": source,
        "from": translator
    }
    return result

ultra_vi = load_dataset("nguyenthanhdo/ultrachat-75k-vi", split="train_sft")
ultra_vi = ultra_vi.remove_columns(["prompt_id", "prompt"])
ultra_vi = ultra_vi.add_column("source", ["ultrachat" for _ in ultra_vi])
ultra_vi = ultra_vi.add_column("from", ["gpt3.5" for _ in ultra_vi])
ultra_vi_z = ultra_vi.map(zephyr_format)

from datasets import load_dataset
viettel = load_dataset("nguyenthanhdo/viettel_v3.2", split="train")
viettel = viettel.add_column("from", ["googletrans" for _ in viettel])

webglm = viettel.filter(lambda x: x["source"] == "webglm")
webglm_z = webglm.map(zephyr_format)

squad = viettel.filter(lambda x: x["source"] == "squad_paraphrases")
squad_z = squad.map(zephyr_format)

dolphin_cqa = viettel.filter(lambda x: x["source"] == "dolphin_cqa_long")
dolphin_cqa_z = dolphin_cqa.map(zephyr_format)

wizard = viettel.filter(lambda x: x["source"] == "wizard_nocode")
wizard_z = wizard.map(zephyr_format)

ultra_aem = load_dataset("nguyenthanhdo/ultrachat-aem-vi-v2.1", split="train")
ultra_aem = ultra_aem.add_column("source", ["ultra-aem" for _ in ultra_aem])
ultra_aem = ultra_aem.add_column("from", ["googletrans" for _ in ultra_aem])
ultra_aem_z = ultra_aem.map(zephyr_format)

uns = load_dataset("nguyenthanhdo/orca-unanswerable-v2.1", split="vi")
uns_long = uns.filter(lambda x: x["excuse"] == "long_excuse")
uns_long = uns_long.add_column("source", ["null-long" for _ in uns_long])
uns_short = uns.filter(lambda x: x["excuse"] == "short_excuse")
uns_short = uns_short.add_column("source", ["null-short" for _ in uns_short])
uns_long_z = uns_long.map(zephyr_format)
uns_short_z = uns_short.map(zephyr_format)

from datasets import concatenate_datasets
dont_cqa = concatenate_datasets([
    ultra_vi_z,
    squad_z,
    dolphin_cqa_z,
    ultra_aem_z,
    webglm_z,
    uns_long_z,
    uns_short_z,
    wizard_z
])
dont_cqa.push_to_hub("nguyenthanhdo/caphesuada-290k")

"""
    v 50k ultrachat first turn (datht)
    v 10k squad_paraphrases 
    v 15k answerable CQA - dolphin
    v 25k answerable CQA - Ultrachat-AEM
    v 20k multi source CQA - WebGLM
    v 15k for CQA with no answer and long excuse - Orca
    v 15k for CQA with no answer and short excuse - Orca
    v 7.5k for Wizard
"""
seed = 112
tini_cfsd = concatenate_datasets([
    ultra_vi_z.train_test_split(train_size=42500, seed=seed)["train"],
    squad_z, # keep full 10000
    dolphin_cqa_z.train_test_split(train_size=15000, seed=seed)["train"],
    ultra_aem_z.train_test_split(train_size=25000, seed=seed)["train"],
    webglm_z.train_test_split(train_size=20000, seed=seed)["train"],
    uns_long_z.train_test_split(train_size=15000, seed=seed)["train"],
    uns_short_z.train_test_split(train_size=15000, seed=seed)["train"],
    wizard_z # keep full 7500
])
tini_cfsd.push_to_hub("nguyenthanhdo/caphesuada-150k")
```