size_categories:10K<n<100K format:parquet modality:text library:datasets library:pandas library:mlcroissant library:polars region:us

---
dataset_info:
- config_name: ArLegalBench
  features:
  - name: index
    dtype: string
  - name: question_english
    dtype: string
  - name: answer_english
    dtype: string
  - name: contract_english
    dtype: string
  - name: Question
    dtype: string
  - name: Answer
    dtype: string
  - name: Context
    dtype: string
  - name: choices
    sequence: string
  - name: choices_english
    sequence: string
  - name: subject
    dtype: string
  - name: task
    dtype: string
  splits:
  - name: test
    num_bytes: 19243593
    num_examples: 15311
  - name: train
    num_bytes: 51939
    num_examples: 28
  download_size: 3389747
  dataset_size: 19295532
- config_name: MCQs
  features:
  - name: Engine
    dtype: string
  - name: Context
    dtype: string
  - name: Question
    dtype: string
  - name: Answer
    dtype: string
  - name: Option 1
    dtype: string
  - name: Option 2
    dtype: string
  - name: Option 3
    dtype: string
  - name: Option 4
    dtype: string
  - name: Answer Key
    dtype: int64
  - name: task
    dtype: string
  - name: subject
    dtype: string
  - name: choices
    sequence: string
  splits:
  - name: train
    num_bytes: 12185640.889357671
    num_examples: 9291
  - name: test
    num_bytes: 3046738.1106423284
    num_examples: 2323
  download_size: 7038064
  dataset_size: 15232379.0
- config_name: QA
  features:
  - name: Question
    dtype: string
  - name: Answer
    dtype: string
  - name: Context
    dtype: string
  - name: task
    dtype: string
  - name: subject
    dtype: string
  splits:
  - name: train
    num_bytes: 81006.03797468354
    num_examples: 63
  - name: test
    num_bytes: 20572.962025316454
    num_examples: 16
  download_size: 56179
  dataset_size: 101579.0
configs:
- config_name: ArLegalBench
  data_files:
  - split: train
    path: ArLegalBench/train-*
  - split: test
    path: ArLegalBench/test-*
- config_name: MCQs
  data_files:
  - split: train
    path: MCQs/train-*
  - split: test
    path: MCQs/test-*
- config_name: QA
  data_files:
  - split: train
    path: QA/train-*
  - split: test
    path: QA/test-*
---

# ArabLegalEval

## Populating prompt templates

```py
import random
import requests
import yaml
from datasets import load_dataset

def render_prompt_template(dataset_entry, technique_map):
    subject = dataset_entry['subject']
    task = dataset_entry['task']
    technique = technique_map[task]
    prompt_template = prompts[task][subject][technique]
    prompt_template = (
        prompt_template
        .replace('{correct_answer}', '{Answer}')
        .replace('{question}', '{Question}')
        .replace('{query_str}', '{Question}')
        .replace('{clause}', '{Context}')
        .replace('{context}', '{Context}')
        .replace('{context_str}', '{Context}')
        .replace('{options}', '{choices}')
    )
    assert '{Question}' in prompt_template, f"Question not found in prompt template for {subject}/{task}"
    rendered_instruction = prompt_template.replace('{Question}', dataset_entry['Question'])
    rendered_instruction = rendered_instruction.replace('{Context}', dataset_entry['Context'])
    dataset_entry['rendered_instruction'] = rendered_instruction
    return dataset_entry

def populate_choices(dataset_entry):
    assert 'rendered_instruction' in dataset_entry, f"rendered_instruction not found for {subject}/{task}"
    subject = dataset_entry['subject']
    task = dataset_entry['task']
    if '{choices}' not in dataset_entry['rendered_instruction']:
        return dataset_entry

    if task == "MCQs":
        alpa_en = ['A', 'B', 'C', 'D', 'E']
        # Get the actual option values (not column names) that aren't None/empty
        choices_columns = [
            c for c in
            [
                'Option 1', 'Option 2', 'Option 3', 'Option 4'
            ]
            if c in dataset_entry.keys() and dataset_entry[c] is not None and dataset_entry[c] != ''
        ]
        choices = [
            dataset_entry[c] for c in choices_columns
        ]
        correct_answer_index_before_shuffling = int(dataset_entry['Answer Key']) - 1
        # assert choices[correct_answer_index_before_shuffling] == dataset_entry['Answer'], f"Answer '{dataset_entry['Answer']}' not found at index {correct_answer_index_before_shuffling} in choices {choices}"
        # print("Choices before shuffling:", choices, "Answer Key:", dataset_entry['Answer Key'], "Answer:", dataset_entry['Answer'])
        # Randomly shuffle the choices
        shuffled_indices = list(range(len(choices)))
        random.shuffle(shuffled_indices)
        correct_answer_index_after_shuffling = shuffled_indices.index(correct_answer_index_before_shuffling)
        shuffled_choices = [choices[i] for i in shuffled_indices]
        choices_str = '\n'.join([f"{alpa_en[i]}. {choice}" for i, choice in enumerate(shuffled_choices)])
        dataset_entry['Answer Key'] = alpa_en[correct_answer_index_after_shuffling]
        dataset_entry['rendered_instruction'] = dataset_entry['rendered_instruction'].format(choices=choices_str)
        dataset_entry['Answer'] = shuffled_choices[correct_answer_index_after_shuffling]
        assert shuffled_choices[correct_answer_index_after_shuffling] == dataset_entry['Answer'], f"Answer '{dataset_entry['Answer']}' not found at index {correct_answer_index_after_shuffling} in shuffled choices {shuffled_choices}"
        assert dataset_entry['Answer Key'] == alpa_en[correct_answer_index_after_shuffling], f"Answer Key '{dataset_entry['Answer Key']}' not found in shuffled choices {shuffled_choices}"
        index_from_alpha = alpa_en.index(dataset_entry['Answer Key'])
        assert shuffled_choices[index_from_alpha] == dataset_entry['Answer'], f"Answer '{dataset_entry['Answer']}' not found in shuffled choices {shuffled_choices}"
    elif task == "ArLegalBench":
        raise Exception("ArLegalBench is not supported")
    else:
        raise ValueError(f"Unsupported task: {task}")

    return dataset_entry


# download and load prompts.yaml
prompts = yaml.safe_load(requests.get("https://huggingface.co/datasets/THIQAH-Research/ArabLegalEval/resolve/main/prompts.yaml?download=true").content)

## feel free to change the technique map to your liking
fn_kwargs = {
    "technique_map": {
        "QA": "TEXT_QA_TEMPLATE",
        "MCQs": "MCQ_PROMPT_WITH_CONTEXT",
        "ArLegalBench": "Zero_shot"
    }
}

arabLegalEvalDatasetKeys = ["MCQs", "QA", "ArLegalBench"]
arabLegalEval = {
    key: load_dataset("THIQAH-Research/ArabLegalEval", key) for key in arabLegalEvalDatasetKeys
}

for name, ds in arabLegalEval.items():
    for split in ds.keys():
        ds[split] = ds[split].map(render_prompt_template, fn_kwargs=fn_kwargs)
        ds[split] = ds[split].map(populate_choices)
    x = ds["test"][0]
    print(f'rendered instruction for {name}:\n' + x['rendered_instruction'], "Answer:", x['Answer'])
```

```
rendered instruction for MCQs:
هذا سؤال قانوني للعامة في المملكة العربية السعودية. اختر الإجابة الصحيحة!

سياق: يكون تبليغ غير السعودي المتحقق وجوده داخل المملكة وليس له مكان إقامة معروف وفق الفقرة (ط) من المادة السابعة عشرة من هذا النظام.
سؤال: ما هو شرط تبليغ غير السعودي وفق الفقرة (ط) من المادة السابعة عشرة من هذا النظام؟
A. أن يكون متحقق وجوده خارج المملكة وليس له مكان إقامة معروف
B. أن يكون متحقق وجوده داخل المملكة وله مكان إقامة معروف
C. أن يكون متحقق وجوده خارج المملكة وله مكان إقامة معروف
D. أن يكون متحقق وجوده داخل المملكة وليس له مكان إقامة معروف

إجابة: 
 Answer: أن يكون متحقق وجوده داخل المملكة وليس له مكان إقامة معروف
```
