size_categories:100K<n<1M format:parquet modality:tabular modality:text library:datasets library:dask library:mlcroissant library:polars region:us
---
dataset_info:
  features:
  - name: conversations
    list:
    - name: from
      dtype: string
    - name: value
      dtype: string
  - name: shard_id
    dtype: string
  - name: output
    dtype: string
  - name: ngram_3_uniqueness
    dtype: float64
  - name: entropy
    dtype: float64
  - name: gini_index
    dtype: float64
  - name: self_bleu
    dtype: float64
  - name: embeddings
    dtype: string
  - name: kmeans_inertia_embeddings
    dtype: float64
  - name: kmeans_inertia_embeddings_normalized
    dtype: float64
  - name: projected_gradients_embeddings
    dtype: float64
  - name: new_conversations
    dtype: string
  - name: projected_gradients
    dtype: string
  - name: projected_gradients_vendi
    dtype: float64
  - name: projected_gradients_log_det
    dtype: float64
  - name: projected_embeddings_log_det
    dtype: 'null'
  - name: kmeans_inertia_gradients
    dtype: float64
  - name: kmeans_inertia_gradients_normalized
    dtype: float64
  splits:
  - name: train
    num_bytes: 13192221325
    num_examples: 863286
  download_size: 8045826625
  dataset_size: 13192221325
configs:
- config_name: default
  data_files:
  - split: train
    path: data/train-*
---
