size_categories:n<1K format:parquet modality:tabular modality:text library:datasets library:pandas library:mlcroissant library:polars region:us
---
dataset_info:
  features:
  - name: file_id
    dtype: string
  - name: content
    dtype: string
  - name: repo
    dtype: string
  - name: path
    dtype: string
  - name: token_length
    dtype: int64
  - name: original_comment
    dtype: string
  - name: comment_type
    dtype: string
  - name: detected_lang
    dtype: string
  - name: masked_comment
    dtype: string
  - name: excluded
    dtype: bool
  - name: file-tokens-Qwen/Qwen2-7B
    dtype: int64
  - name: comment-tokens-Qwen/Qwen2-7B
    dtype: int64
  - name: file-tokens-bigcode/starcoder2-7b
    dtype: int64
  - name: comment-tokens-bigcode/starcoder2-7b
    dtype: int64
  - name: file-tokens-google/codegemma-7b
    dtype: int64
  - name: comment-tokens-google/codegemma-7b
    dtype: int64
  - name: file-tokens-ibm-granite/granite-8b-code-base
    dtype: int64
  - name: comment-tokens-ibm-granite/granite-8b-code-base
    dtype: int64
  - name: file-tokens-meta-llama/CodeLlama-7b-hf
    dtype: int64
  - name: comment-tokens-meta-llama/CodeLlama-7b-hf
    dtype: int64
  - name: excluded-based-on-tokenizer-Qwen/Qwen2-7B
    dtype: bool
  - name: excluded-based-on-tokenizer-bigcode/starcoder2-7b
    dtype: bool
  - name: excluded-based-on-tokenizer-google/codegemma-7b
    dtype: bool
  - name: excluded-based-on-tokenizer-ibm-granite/granite-8b-code-base
    dtype: bool
  - name: excluded-based-on-tokenizer-meta-llama/CodeLlama-7b-hf
    dtype: bool
  - name: include-for-inference
    dtype: bool
  splits:
  - name: train
    num_bytes: 6422752
    num_examples: 800
  download_size: 2634846
  dataset_size: 6422752
configs:
- config_name: default
  data_files:
  - split: train
    path: data/train-*
---
