size_categories:10K<n<100K format:parquet modality:tabular modality:text library:datasets library:pandas library:mlcroissant library:polars region:us
---
configs:
- config_name: default
  data_files:
  - split: en
    path: data/en-*
  - split: vi
    path: data/vi-*
dataset_info:
  features:
  - name: id
    dtype: string
  - name: system_prompt
    dtype: string
  - name: question
    dtype: string
  - name: response
    dtype: string
  - name: question_word_count
    dtype: int64
  - name: response_word_count
    dtype: int64
  - name: from
    dtype: string
  - name: excuse
    dtype: string
  splits:
  - name: en
    num_bytes: 105709660.64824432
    num_examples: 40274
  - name: vi
    num_bytes: 138263396.8695503
    num_examples: 40274
  download_size: 131306825
  dataset_size: 243973057.5177946
---
# Dataset Card for "orca-unanswerable-v2.1"

```python
from datasets import load_dataset
from datasets import concatenate_datasets

unanswerable = load_dataset("nguyenthanhdo/orca-unanswerable-v2.0") ## this is an english dataset

splits = ["long_excuse", "short_excuse"]
dss = []
for split in splits:
    ds = unanswerable[split]
    ds = ds.add_column("excuse", [split for _ in ds])
    dss.append(ds)
uns_en2 = concatenate_datasets(dss)

def exclude_summ(example):
    """
    It shows that quetions about names do not always work.
    We also do not want summaries
    We also do not want too long responses, since it's a sign of ranting with irrelevant info.
    """
    keywords = ["summ", "main point", "first name", "last name", "full name"]
    flag = True
    if any([k in example["question"].lower() for k in keywords]):
        flag = False
    if example["response_word_count"] > 150:
        flag = False
    if example["question_word_count"] > 2000:
        flag = False
    return {"flag": flag}
    # return flag

uns_en_filtered = uns_en2.map(exclude_summ)
x = uns_en_filtered.filter(lambda x: x["flag"])
x = x.remove_columns(["flag"])

## We translate nguyenthanhdo/orca-unanswerable-v2.0 and do the same for the vi split.
"..."
y = uns_vi_filtered.filter(lambda x: x["flag"])
y = y.remove_columns(["flag"])

uns_v2 = DatasetDict({
    "en": x, "vi": y
})
uns_v2.push_to_hub("nguyenthanhdo/orca-unanswerable-v2.1")
```