size_categories:n<1K format:parquet modality:tabular modality:text library:datasets library:pandas library:mlcroissant library:polars region:us
---
dataset_info:
- config_name: java
  features:
  - name: id
    dtype: int64
  - name: text_id
    dtype: string
  - name: repo_owner
    dtype: string
  - name: repo_name
    dtype: string
  - name: issue_url
    dtype: string
  - name: pull_url
    dtype: string
  - name: comment_url
    dtype: string
  - name: links_count
    dtype: int64
  - name: link_keyword
    dtype: string
  - name: issue_title
    dtype: string
  - name: issue_body
    dtype: string
  - name: base_sha
    dtype: string
  - name: head_sha
    dtype: string
  - name: diff_url
    dtype: string
  - name: diff
    dtype: string
  - name: changed_files
    dtype: string
  - name: changed_files_exts
    dtype: string
  - name: changed_files_count
    dtype: int64
  - name: java_changed_files_count
    dtype: int64
  - name: kt_changed_files_count
    dtype: int64
  - name: py_changed_files_count
    dtype: int64
  - name: code_changed_files_count
    dtype: int64
  - name: repo_symbols_count
    dtype: int64
  - name: repo_tokens_count
    dtype: int64
  - name: repo_lines_count
    dtype: int64
  - name: repo_files_without_tests_count
    dtype: int64
  - name: changed_symbols_count
    dtype: int64
  - name: changed_tokens_count
    dtype: int64
  - name: changed_lines_count
    dtype: int64
  - name: changed_files_without_tests_count
    dtype: int64
  - name: issue_symbols_count
    dtype: int64
  - name: issue_words_count
    dtype: int64
  - name: issue_tokens_count
    dtype: int64
  - name: issue_lines_count
    dtype: int64
  - name: issue_links_count
    dtype: int64
  - name: issue_code_blocks_count
    dtype: int64
  - name: pull_create_at
    dtype: timestamp[s]
  - name: repo_stars
    dtype: int64
  - name: repo_language
    dtype: string
  - name: repo_languages
    dtype: string
  - name: repo_license
    dtype: string
  splits:
  - name: test
    num_bytes: 675109
    num_examples: 50
  download_size: 243234
  dataset_size: 675109
- config_name: kt
  features:
  - name: id
    dtype: int64
  - name: text_id
    dtype: string
  - name: repo_owner
    dtype: string
  - name: repo_name
    dtype: string
  - name: issue_url
    dtype: string
  - name: pull_url
    dtype: string
  - name: comment_url
    dtype: string
  - name: links_count
    dtype: int64
  - name: link_keyword
    dtype: string
  - name: issue_title
    dtype: string
  - name: issue_body
    dtype: string
  - name: base_sha
    dtype: string
  - name: head_sha
    dtype: string
  - name: diff_url
    dtype: string
  - name: diff
    dtype: string
  - name: changed_files
    dtype: string
  - name: changed_files_exts
    dtype: string
  - name: changed_files_count
    dtype: int64
  - name: java_changed_files_count
    dtype: int64
  - name: kt_changed_files_count
    dtype: int64
  - name: py_changed_files_count
    dtype: int64
  - name: code_changed_files_count
    dtype: int64
  - name: repo_symbols_count
    dtype: int64
  - name: repo_tokens_count
    dtype: int64
  - name: repo_lines_count
    dtype: int64
  - name: repo_files_without_tests_count
    dtype: int64
  - name: changed_symbols_count
    dtype: int64
  - name: changed_tokens_count
    dtype: int64
  - name: changed_lines_count
    dtype: int64
  - name: changed_files_without_tests_count
    dtype: int64
  - name: issue_symbols_count
    dtype: int64
  - name: issue_words_count
    dtype: int64
  - name: issue_tokens_count
    dtype: int64
  - name: issue_lines_count
    dtype: int64
  - name: issue_links_count
    dtype: int64
  - name: issue_code_blocks_count
    dtype: int64
  - name: pull_create_at
    dtype: timestamp[s]
  - name: repo_stars
    dtype: int64
  - name: repo_language
    dtype: string
  - name: repo_languages
    dtype: string
  - name: repo_license
    dtype: string
  splits:
  - name: test
    num_bytes: 424086
    num_examples: 50
  download_size: 183832
  dataset_size: 424086
- config_name: py
  features:
  - name: id
    dtype: int64
  - name: text_id
    dtype: string
  - name: repo_owner
    dtype: string
  - name: repo_name
    dtype: string
  - name: issue_url
    dtype: string
  - name: pull_url
    dtype: string
  - name: comment_url
    dtype: string
  - name: links_count
    dtype: int64
  - name: link_keyword
    dtype: string
  - name: issue_title
    dtype: string
  - name: issue_body
    dtype: string
  - name: base_sha
    dtype: string
  - name: head_sha
    dtype: string
  - name: diff_url
    dtype: string
  - name: diff
    dtype: string
  - name: changed_files
    dtype: string
  - name: changed_files_exts
    dtype: string
  - name: changed_files_count
    dtype: int64
  - name: java_changed_files_count
    dtype: int64
  - name: kt_changed_files_count
    dtype: int64
  - name: py_changed_files_count
    dtype: int64
  - name: code_changed_files_count
    dtype: int64
  - name: repo_symbols_count
    dtype: int64
  - name: repo_tokens_count
    dtype: int64
  - name: repo_lines_count
    dtype: int64
  - name: repo_files_without_tests_count
    dtype: int64
  - name: changed_symbols_count
    dtype: int64
  - name: changed_tokens_count
    dtype: int64
  - name: changed_lines_count
    dtype: int64
  - name: changed_files_without_tests_count
    dtype: int64
  - name: issue_symbols_count
    dtype: int64
  - name: issue_words_count
    dtype: int64
  - name: issue_tokens_count
    dtype: int64
  - name: issue_lines_count
    dtype: int64
  - name: issue_links_count
    dtype: int64
  - name: issue_code_blocks_count
    dtype: int64
  - name: pull_create_at
    dtype: timestamp[s]
  - name: repo_stars
    dtype: int64
  - name: repo_language
    dtype: string
  - name: repo_languages
    dtype: string
  - name: repo_license
    dtype: string
  splits:
  - name: test
    num_bytes: 328403
    num_examples: 50
  download_size: 168497
  dataset_size: 328403
configs:
- config_name: java
  data_files:
  - split: test
    path: java/test-*
- config_name: kt
  data_files:
  - split: test
    path: kt/test-*
- config_name: py
  data_files:
  - split: test
    path: py/test-*
---
# BenchName (Bug localization)

This is the benchmark for the Bug localization task as part of the
[BenchName benchmark](https://huggingface.co/collections/icmlbenchname/icml-25-benchname-679b57e38a389f054cdf33e9).


The bug localization problem can be formulated as follows: given an issue with a bug description and a repository snapshot in a state where the bug is reproducible, identify the files within the repository that need to be modified to address the reported bug.

The dataset provides all the required components for evaluation of bug localization approaches in real project-level large-scale data collected from GitHub, including:
* Bug issue description;
* Repositories, from which the content can be extracted at the state of the commit SHA where the bug is reproducible;
* List of files that should be changed in order to solve the bug;
* Other additional data and metrics that can be useful in developing new approaches.

All the repositories are published under permissive licenses (MIT, Apache-2.0, BSD-3-Clause, and BSD-2-Clause). The datapoints can be removed upon request.

The collected dataset was carefully filtered, enhanced with useful metrics and, what's more, manually labeled, which assures the data quality and provides a golden subset of good examples for evaluation.\
Moreover, the dataset was split into several categories, namely:

| **Category**   |  **Description**  |  **Number of data points**  |
|:------------------:|:----------------------------------------:|:----------------------------------------:|
| `py` | Only `.py` files in changes | 4,339 |
| `java` | Only `.java` files in changes | 2,522  | 
| `kt`  | Only `.kt` files in changes | 618  |

...and splits, namely:
| **Split**   |  **Description**  |
|:------------------:|:----------------------------------------:|
| `dev` | All collected datapoints |
| `test` | Manually verified datapoints |
| `train` | Rest of the datapoint from `dev` without `test` | 

The results of evaluation of various bug localization approaches can be found in the [BenchName leaderboard](https://huggingface.co/collections/icmlbenchname/icml-25-benchname-679b57e38a389f054cdf33e9).

The following sections describe the utilities around the dataset, as well as dataset content.

## How-to

* Load the data via [`load_dataset`](https://huggingface.co/docs/datasets/v2.14.3/en/package_reference/loading_methods#datasets.load_dataset):

    ```py
    from datasets import load_dataset
    
    # Select a configuration from ["py", "java", "kt"]
    configuration = "py"
    # Select a split from ["dev", "train", "test"]
    split = "dev"
    # Load data
    dataset = load_dataset("icmlbenchname/bug-localization", configuration, split=split)  
    ```

* Load repos via [`hf_hub_download`](https://huggingface.co/docs/huggingface_hub/v0.20.3/en/package_reference/file_download#huggingface_hub.hf_hub_download):
    ```py
    from huggingface_hub import hf_hub_download
    from datasets import load_dataset
    import zipfile
    import os
    
    # Load json with list of repos' .tar.gz file paths
    paths_json = load_dataset("icmlbenchname/bug-localization", data_files="repos.json")

    # Load each repo in .tar.gz format, unzip, delete archive
    repos = paths_json[configuration][0]
    repos_path = "path/to/repos"

    for i, repo_zip_path in enumerate(repos):
        repo_name = os.path.basename(repo_zip_path).split('.')[0]
        repo_path = os.path.join(repos_path, repo_name)

        # Load repo zip
        local_repo_zip_path = hf_hub_download(
            "icmlbenchname/bug-localization",
            filename=repo_zip_path,
            repo_type="dataset",
            local_dir="path/to/zips"
        )

        # Unzip repo
        with zipfile.ZipFile(local_repo_zip_path, 'r') as zip_ref:
            zip_ref.extractall(repo_path)
        os.remove(local_repo_zip_path)
   ```

* Data streaming via [HFDataSource](https://anonymous.4open.science/r/icml-benchname-2025/bug_localization/src/baselines/data_sources/hf_data_source.py). 
  Besides data loading, `HFDataSource` returns a datapoint for running the baseline along with the content of the repository at the state where the bug is reproducible (aka `base_sha` commit of the pull request that resloves the bug issue). \
  All source code for working with the Git history of repositories (commits, diffs, etc.) is available in [`git_utils.py`](https://anonymous.4open.science/r/icml-benchname-2025/bug_localization/src/utils/git_utils.py), as is an example of baselines, utilizing this dataset.

### Bug localization data

Each datapoint contains the main fields, as well as additional metrics calculated on them. \
The main fields are:

|     **Field**      |             **Description**              |
|:------------------:|:----------------------------------------:|
|    `repo_owner`    |            Owner of the repository with the bug issue.            |
|    `repo_name`    |            Name of the repository with the bug issue.            |
|    `issue_url`      |       GitHub link to the issue <br> `https://github.com/{repo_owner}/{repo_name}/issues/{issue_id}`.       |
|    `pull_url`      |       GitHub link to the pull request <br> `https://github.com/{repo_owner}/{repo_name}/pull/{pull_id}`.       |
|    `comment_url`      |       GitHub link to the comment with a reference from pull request to issue <br> `https://github.com/{repo_owner}/{repo_name}/pull/{pull_id}#issuecomment-{comment_id}`.       |
|    `issue_title`       | Issue title.  |
|    `issue_body`       | Issue body.  |
|    `base_sha`       | Base SHA of the pull request.  |
|    `head_sha`        | Head SHA of the pull request.  |
|    `diff_url`        | Link to the diff between the base and the head SHA <br> `https://github.com/{repo_owner}/{repo_name}/compare/{base_sha}...{head_sha}`.  |
|    `diff`       | Content of the diff.  |
|    `pull_create_at`       | Date of pull request creation in the `yyyy-mm-ddThh:mm:ssZ` format.  |
|    `repo_stars`       | Number of stars of the repo.  |
|    `changed_files`*       | List of the changed files parsed from diff.  |
|    `repo_language`       | Main programming language used in the repository.  |
|    `repo_languages`       | All programming languages used in the repository.  |
|    `repo_license`       | License assigned to the repository.  |

\* Excluding test files that do not contain bug causes, rather changed in order to add tests for proving that the bug is gone.

The metrics-related fields are:
|     **Field**      |             **Description**              |
|:------------------:|:----------------------------------------:|
|    `changed_files_exts`       | Dictionary from the extension of changed files to their count.  |
|    `changed_files_count`       | Number of changed files.  |
|    `java_changed_files_count`       | Number of changed `.java` files.  |
|    `kt_changed_files_count`       | Number of changed `.kt` files.  |
|    `py_changed_files_count`       | Number of changed `.py` files.  |
|    `code_changed_files_count`       | Number of changed `.java`, `.kt`, or `.py` files. |
|    `repo_symbols_count`*       | Number of symbols in the repository. |
|    `repo_tokens_count`*       | Number of tokens** in the files of the repository. |
|    `repo_lines_count`*       | Number of lines in the files of the repository. |
|    `repo_files_without_tests_count`*       | Number of files in the repository. |
|    `changed_symbols_count`*       | Number of symbols in the changed lines in diff. |
|    `changed_tokens_count`*       | Number of tokens** in the changed lines in diff. |
|    `changed_lines_count`*       | Number of changed lines in diff (including added and deleted). |
|    `changed_files_without_tests_count`*       | Number of files in diff. |
|    `issue_symbols_count`*       | Number of symbols in the issue body. |
|    `issue_words_count`*       | Number of words in the issue body (separated by space symbols). |
|    `issue_tokens_count`*       | Number of tokens** in the issue body. |
|    `issue_lines_count`*       | Number of text lines in the issue body (separated by `\\n`). |
|    `issue_links_count`       | Number of links (\[...\](...)) present in the issue body. |
|    `issue_code_blocks_count`       | Number of code blocks (\`\`\`...\`\`\`) present in the issue body. |
            
\* Excluding test files that do not contain bug causes, rather changed in order to add tests for proving that the bug is gone. \
\*\* Using GPT-4 tokenizer via ticktoken.
