license:cc0-1.0 size_categories:10K<n<100K format:parquet modality:text library:datasets library:dask library:mlcroissant library:polars region:us
---
license: cc0-1.0
dataset_info:
- config_name: 100M_1
  features:
  - name: text
    dtype: string
  - name: title
    dtype: string
  - name: author
    dtype: string
  - name: author year of birth
    dtype: string
  - name: author year of death
    dtype: string
  - name: language
    dtype: string
  - name: downloads
    dtype: string
  - name: subjects
    dtype: string
  - name: document id
    dtype: string
  - name: type
    dtype: string
  splits:
  - name: train
    num_bytes: 583690713
    num_examples: 1551
  - name: validation
    num_bytes: 37679070
    num_examples: 100
  download_size: 379041407
  dataset_size: 621369783
- config_name: 100M_2
  features:
  - name: text
    dtype: string
  - name: title
    dtype: string
  - name: author
    dtype: string
  - name: author year of birth
    dtype: string
  - name: author year of death
    dtype: string
  - name: language
    dtype: string
  - name: downloads
    dtype: string
  - name: subjects
    dtype: string
  - name: document id
    dtype: string
  - name: type
    dtype: string
  splits:
  - name: train
    num_bytes: 587333788
    num_examples: 1640
  - name: validation
    num_bytes: 35136453
    num_examples: 100
  download_size: 378535463
  dataset_size: 622470241
- config_name: 100M_3
  features:
  - name: text
    dtype: string
  - name: title
    dtype: string
  - name: author
    dtype: string
  - name: author year of birth
    dtype: string
  - name: author year of death
    dtype: string
  - name: language
    dtype: string
  - name: downloads
    dtype: string
  - name: subjects
    dtype: string
  - name: document id
    dtype: string
  - name: type
    dtype: string
  splits:
  - name: train
    num_bytes: 584453507
    num_examples: 1597
  - name: validation
    num_bytes: 43374300
    num_examples: 100
  download_size: 384166106
  dataset_size: 627827807
- config_name: 10M_1
  features:
  - name: text
    dtype: string
  - name: title
    dtype: string
  - name: author
    dtype: string
  - name: author year of birth
    dtype: string
  - name: author year of death
    dtype: string
  - name: language
    dtype: string
  - name: downloads
    dtype: string
  - name: subjects
    dtype: string
  - name: document id
    dtype: string
  - name: type
    dtype: string
  splits:
  - name: train
    num_bytes: 57895123
    num_examples: 163
  - name: validation
    num_bytes: 37761505
    num_examples: 100
  download_size: 58641601
  dataset_size: 95656628
- config_name: 10M_2
  features:
  - name: text
    dtype: string
  - name: title
    dtype: string
  - name: author
    dtype: string
  - name: author year of birth
    dtype: string
  - name: author year of death
    dtype: string
  - name: language
    dtype: string
  - name: downloads
    dtype: string
  - name: subjects
    dtype: string
  - name: document id
    dtype: string
  - name: type
    dtype: string
  splits:
  - name: train
    num_bytes: 58456082
    num_examples: 143
  - name: validation
    num_bytes: 39871126
    num_examples: 100
  download_size: 59385693
  dataset_size: 98327208
- config_name: 10M_3
  features:
  - name: text
    dtype: string
  - name: title
    dtype: string
  - name: author
    dtype: string
  - name: author year of birth
    dtype: string
  - name: author year of death
    dtype: string
  - name: language
    dtype: string
  - name: downloads
    dtype: string
  - name: subjects
    dtype: string
  - name: document id
    dtype: string
  - name: type
    dtype: string
  splits:
  - name: train
    num_bytes: 58248419
    num_examples: 171
  - name: validation
    num_bytes: 33606180
    num_examples: 100
  download_size: 55895614
  dataset_size: 91854599
- config_name: all
  features:
  - name: text
    dtype: string
  - name: title
    dtype: string
  - name: author
    dtype: string
  - name: author year of birth
    dtype: string
  - name: author year of death
    dtype: string
  - name: language
    dtype: string
  - name: downloads
    dtype: string
  - name: subjects
    dtype: string
  - name: document id
    dtype: string
  - name: type
    dtype: string
  splits:
  - name: train
    num_bytes: 21761545401
    num_examples: 58477
  download_size: 13237223526
  dataset_size: 21761545401
configs:
- config_name: 100M_1
  data_files:
  - split: train
    path: 100M_1/train-*
  - split: validation
    path: 100M_1/validation-*
- config_name: 100M_2
  data_files:
  - split: train
    path: 100M_2/train-*
  - split: validation
    path: 100M_2/validation-*
- config_name: 100M_3
  data_files:
  - split: train
    path: 100M_3/train-*
  - split: validation
    path: 100M_3/validation-*
- config_name: 10M_1
  data_files:
  - split: train
    path: 10M_1/train-*
  - split: validation
    path: 10M_1/validation-*
- config_name: 10M_2
  data_files:
  - split: train
    path: 10M_2/train-*
  - split: validation
    path: 10M_2/validation-*
- config_name: 10M_3
  data_files:
  - split: train
    path: 10M_3/train-*
  - split: validation
    path: 10M_3/validation-*
- config_name: all
  data_files:
  - split: train
    path: all/train-*
---
Number of examples in each subset:

* `all`: 58477 (full dataset)
* `10M_1`: 162
* `10M_2`: 142
* `10M_3`: 170
* `100M_1`: 1550
* `100M_2`: 1639
* `100M_3`: 1596