size_categories:100K<n<1M format:parquet modality:text library:datasets library:pandas library:mlcroissant library:polars region:us
---
dataset_info:
- config_name: 1k-token
  features:
  - name: uid
    dtype: large_string
  - name: english
    dtype: large_string
  - name: japanese
    dtype: large_string
  splits:
  - name: train
    num_bytes: 5411157.601993782
    num_examples: 14919
  - name: test
    num_bytes: 1352154.6712402187
    num_examples: 3728
  download_size: 4286204
  dataset_size: 6763312.273234001
- config_name: 256-token
  features:
  - name: uid
    dtype: large_string
  - name: english
    dtype: large_string
  - name: japanese
    dtype: large_string
  splits:
  - name: train
    num_bytes: 5409344.08982742
    num_examples: 14914
  - name: test
    num_bytes: 1351791.9688069462
    num_examples: 3727
  download_size: 4277596
  dataset_size: 6761136.058634367
- config_name: 2k-token
  features:
  - name: uid
    dtype: large_string
  - name: english
    dtype: large_string
  - name: japanese
    dtype: large_string
  splits:
  - name: train
    num_bytes: 5411883.006860328
    num_examples: 14921
  - name: test
    num_bytes: 1352517.3736734912
    num_examples: 3729
  download_size: 4301712
  dataset_size: 6764400.380533819
- config_name: 4k-token
  features:
  - name: uid
    dtype: large_string
  - name: english
    dtype: large_string
  - name: japanese
    dtype: large_string
  splits:
  - name: train
    num_bytes: 5412245.7092936
    num_examples: 14922
  - name: test
    num_bytes: 1353605.4809733091
    num_examples: 3732
  download_size: 4324607
  dataset_size: 6765851.190266909
- config_name: 512-token
  features:
  - name: uid
    dtype: large_string
  - name: english
    dtype: large_string
  - name: japanese
    dtype: large_string
  splits:
  - name: train
    num_bytes: 5410432.197127238
    num_examples: 14917
  - name: test
    num_bytes: 1352154.6712402187
    num_examples: 3728
  download_size: 4281776
  dataset_size: 6762586.868367457
- config_name: default
  features:
  - name: uid
    dtype: large_string
  - name: english
    dtype: large_string
  - name: japanese
    dtype: large_string
  splits:
  - name: train
    num_bytes: 5413696.519026691
    num_examples: 14926
  - name: test
    num_bytes: 1353605.4809733091
    num_examples: 3732
  download_size: 4406601
  dataset_size: 6767302.0
configs:
- config_name: 1k-token
  data_files:
  - split: train
    path: 1k-token/train-*
  - split: test
    path: 1k-token/test-*
- config_name: 256-token
  data_files:
  - split: train
    path: 256-token/train-*
  - split: test
    path: 256-token/test-*
- config_name: 2k-token
  data_files:
  - split: train
    path: 2k-token/train-*
  - split: test
    path: 2k-token/test-*
- config_name: 4k-token
  data_files:
  - split: train
    path: 4k-token/train-*
  - split: test
    path: 4k-token/test-*
- config_name: 512-token
  data_files:
  - split: train
    path: 512-token/train-*
  - split: test
    path: 512-token/test-*
- config_name: default
  data_files:
  - split: train
    path: data/train-*
  - split: test
    path: data/test-*
---
