size_categories:n<1K format:parquet modality:text library:datasets library:dask library:mlcroissant library:polars region:us
---
dataset_info:
- config_name: huge_context
  features:
  - name: repo
    dtype: string
  - name: commit_hash
    dtype: string
  - name: completion_file
    struct:
    - name: filename
      dtype: string
    - name: content
      dtype: string
  - name: completion_lines
    struct:
    - name: commited
      sequence: int64
    - name: common
      sequence: int64
    - name: infile
      sequence: int64
    - name: inproject
      sequence: int64
    - name: non_informative
      sequence: int64
    - name: random
      sequence: int64
  - name: repo_snapshot
    sequence:
    - name: filename
      dtype: string
    - name: content
      dtype: string
  - name: completion_lines_raw
    struct:
    - name: commited
      sequence: int64
    - name: common
      sequence: int64
    - name: infile
      sequence: int64
    - name: inproject
      sequence: int64
    - name: non_informative
      sequence: int64
    - name: other
      sequence: int64
  splits:
  - name: test
    num_bytes: 5220294969
    num_examples: 296
  download_size: 1811066263
  dataset_size: 5220294969
- config_name: large_context
  features:
  - name: repo
    dtype: string
  - name: commit_hash
    dtype: string
  - name: completion_file
    struct:
    - name: filename
      dtype: string
    - name: content
      dtype: string
  - name: completion_lines
    struct:
    - name: commited
      sequence: int64
    - name: common
      sequence: int64
    - name: infile
      sequence: int64
    - name: inproject
      sequence: int64
    - name: non_informative
      sequence: int64
    - name: random
      sequence: int64
  - name: repo_snapshot
    sequence:
    - name: filename
      dtype: string
    - name: content
      dtype: string
  - name: completion_lines_raw
    struct:
    - name: commited
      sequence: int64
    - name: common
      sequence: int64
    - name: infile
      sequence: int64
    - name: inproject
      sequence: int64
    - name: non_informative
      sequence: int64
    - name: other
      sequence: int64
  splits:
  - name: test
    num_bytes: 2972051649
    num_examples: 270
  download_size: 1242106504
  dataset_size: 2972051649
- config_name: medium_context
  features:
  - name: repo
    dtype: string
  - name: commit_hash
    dtype: string
  - name: completion_file
    struct:
    - name: filename
      dtype: string
    - name: content
      dtype: string
  - name: completion_lines
    struct:
    - name: commited
      sequence: int64
    - name: common
      sequence: int64
    - name: infile
      sequence: int64
    - name: inproject
      sequence: int64
    - name: non_informative
      sequence: int64
    - name: random
      sequence: int64
  - name: repo_snapshot
    sequence:
    - name: filename
      dtype: string
    - name: content
      dtype: string
  - name: completion_lines_raw
    struct:
    - name: commited
      sequence: int64
    - name: common
      sequence: int64
    - name: infile
      sequence: int64
    - name: inproject
      sequence: int64
    - name: non_informative
      sequence: int64
    - name: other
      sequence: int64
  splits:
  - name: test
    num_bytes: 514963163
    num_examples: 224
  download_size: 225815877
  dataset_size: 514963163
- config_name: small_context
  features:
  - name: repo
    dtype: string
  - name: commit_hash
    dtype: string
  - name: completion_file
    struct:
    - name: filename
      dtype: string
    - name: content
      dtype: string
  - name: completion_lines
    struct:
    - name: commited
      sequence: int64
    - name: common
      sequence: int64
    - name: infile
      sequence: int64
    - name: inproject
      sequence: int64
    - name: non_informative
      sequence: int64
    - name: random
      sequence: int64
  - name: repo_snapshot
    sequence:
    - name: filename
      dtype: string
    - name: content
      dtype: string
  - name: completion_lines_raw
    struct:
    - name: commited
      sequence: int64
    - name: common
      sequence: int64
    - name: infile
      sequence: int64
    - name: inproject
      sequence: int64
    - name: non_informative
      sequence: int64
    - name: other
      sequence: int64
  splits:
  - name: test
    num_bytes: 111028780
    num_examples: 144
  download_size: 37598294
  dataset_size: 111028780
configs:
- config_name: huge_context
  data_files:
  - split: test
    path: huge_context/test-*
- config_name: large_context
  data_files:
  - split: test
    path: large_context/test-*
- config_name: medium_context
  data_files:
  - split: test
    path: medium_context/test-*
- config_name: small_context
  data_files:
  - split: test
    path: small_context/test-*
---

# BenchName (Project-level code completion)
This is the benchmark for Project-level code completion task as part of the [BenchName benchmark](https://huggingface.co/collections/icmlbenchname/icml-25-benchname-679b57e38a389f054cdf33e9).
Each datapoint contains the file for completion, a list of lines to complete with their categories (see the categorization below), and a repository snapshot that can be used to build the context.
All the repositories are published under permissive licenses (MIT, Apache-2.0, BSD-3-Clause, and BSD-2-Clause). The datapoints can be removed upon request.

## How-to

Load the data via [load_dataset](https://huggingface.co/docs/datasets/v2.14.3/en/package_reference/loading_methods#datasets.load_dataset):

```
from datasets import load_dataset

config_names = [
  'small_context',
  'medium_context',
  'large_context',
  'huge_context'
]

ds = load_dataset('icmlbenchname/project-level-code-completion', config_name, split='test')
```
## Dataset Structure

Datapoints in the dataset have the following structure:

* `repo` – repository name in the format `{GitHub_user_name}__{repository_name}`
* `commit_hash` – commit hash of the repository
* `completion_file` – dictionary with the completion file content in the following format:
  * `filename` – path to the completion file
  * `content` – content of the completion file
* `completion_lines` – dictionary where the keys are categories of lines and values are a list of integers (numbers of lines to complete). The categories are:
  * `committed` – line contains at least one function or class from the files that were added on the completion file commit
  * `inproject` – line contains at least one function or class from the repository snapshot at the moment of completion
  * `infile` – line contains at least one function or class from the completion file
  * `common` – line contains at least one function or class with common names, e.g., `main`, `get`, etc.
  * `non_informative` – line that was classified to be non-informative, e.g., too short, contains comments, etc.
  * `random` – other lines.
* `repo_snapshot` – dictionary with a snapshot of the repository before the commit. It has the same structure as `completion_file`, but filenames and contents are orginized as lists.
* `completion_lines_raw` – same as `completion_lines`, but before sampling

## How we collected the data

To collect the data, we cloned repositories from GitHub where the main language is Python.
The completion file for each datapoint is a `.py` file that was added to the repository in a commit.
The state of the repository before this commit is the repo snapshot.

The dataset configurations are based on the number of characters in `.py` files from the repository snapshot:
  - `small_context` – less than 48K characters;
  - `medium_context` – from 48K to 192K characters;
  - `large_context` – from 192K to 768K characters;
  - `huge_context` – more than 768K characters.

## Datasets Stats

| Dataset           | Number of datapoints | Number of repositories | Number of commits |
|:-----------------:|:--------------------:|:----------------------:|:-----------------:|
| `small_context`   | 144                  | 46                     | 63                |
| `medium_context`  | 224                  | 80                     | 175               |
| `large_context`   | 270                  | 75                     | 219               |
| `huge_context`    | 296                  | 75                     | 252               |


### Completion File
|      Dataset       | Completion file lines, min | Completion file lines, max | Completion file lines, median |
|:------------------:|:--------------------------:|:--------------------------:|:------------------------------:|
| `small_context`    |           201            |           1916            |            310.5              |
| `medium_context`   |           200            |           1648            |            310.0              |
| `large_context`    |           200            |           1694            |            278.0              |
| `huge_context`     |           200            |           1877            |            313.5              |



### Repository Snapshot `.py` files
|      Dataset       | Context py files number, min | Context py files number, max | Context py files number, median | Context py lines, median |
|:------------------:|:----------------------------:|:----------------------------:|:--------------------------------:|:-------------------------:|
| `small_context`    |              0               |              52              |                4.0               |           128.0          |
| `medium_context`   |              3               |             117              |               34.0               |          3786.0          |
| `large_context`    |              3               |             255              |               84.0               |         15466.5          |
| `huge_context`     |             47               |            5227              |              261.0               |         49811.0          |


### Repository Snapshot non `.py` files
|      Dataset       | Context non-py files number, min | Context non-py files number, max | Context non-py files number, median | Context non-py lines, median |
|:------------------:|:---------------------------------:|:---------------------------------:|:-----------------------------------:|:-----------------------------:|
| `small_context`    |                 1                 |               1044                |                19.5                |            1227.0            |
| `medium_context`   |                 3                 |               3977                |                64.5                |            9735.0            |
| `large_context`    |                 8                 |               2174                |               155.0                |           18759.0            |
| `huge_context`     |                24                 |               7687                |               262.0                |           60163.0            |


### Line Counts:
|      Dataset       | *infile* | *inproject* | *common* | *commited* | *non-informative* | *random* | **all** |
|:------------------:|:--------:|:-----------:|:--------:|:----------:|:-----------------:|:--------:|:-----:|
| `small_context`    | 1430     | 95          | 500      | 1426       | 532               | 703      | **4686**  |
| `medium_context`   | 2224     | 2236        | 779      | 1495       | 858               | 1084     | **8676**  |
| `large_context`    | 2691     | 2595        | 693      | 1322       | 1019              | 1311     | **9631**  |
| `huge_context`     | 2608     | 2901        | 692      | 1019       | 1164              | 1426     | **9810**  |


## Scores
You can find the results of running various models on this dataset in our [leaderboard](https://huggingface.co/collections/icmlbenchname/icml-25-benchname-679b57e38a389f054cdf33e9).
