{"action":"created","comment":{"url":"https://api.github.com/repos/allenai/brat/pulls/comments/118787001","pull_request_review_id":40616389,"id":118787001,"diff_hunk":"@@ -0,0 +1,230 @@\n+#!/usr/bin/env python\n+# 0-*- Mode: Python; tab-width: 4; indent-tabs-mode: nil; coding: utf-8; -*-\n+\n+import sys\n+import os.path\n+import spacy\n+import itertools\n+import glob\n+import re\n+from sys import path as sys_path\n+\n+# this seems to be necessary for annotations to find its config\n+sys_path.append(os.path.join(os.path.dirname(__file__), '..'))\n+try:\n+    import annotation\n+except ImportError:\n+    # Guessing that we might be in the brat tools/ directory ...\n+    sys_path.append(os.path.join(os.path.dirname(__file__), '../server/src'))\n+    import annotation\n+\n+NLP = spacy.load(\"en\")\n+\n+\n+class EnhancedAnnotatedDoc:\n+    def __init__(self, text_annotation):\n+        self.brat_annotation = text_annotation\n+        self.spacy_doc = NLP(text_annotation.get_document_text())\n+        required_boundaries = self._get_entity_boundaries_for_tokenization()\n+        self._impose_token_boundaries(required_boundaries)\n+        self.document_id = os.path.basename(text_annotation.get_document())\n+        self.annotator_id = os.path.basename(os.path.dirname(text_annotation.get_document()))\n+        self._regenerate_from_brat()\n+\n+    def copy_with_compatible_tokenization(self, boundaries):\n+        copy = EnhancedAnnotatedDoc(self.brat_annotation)\n+        copy._impose_token_boundaries(boundaries)\n+        copy._regenerate_from_brat()\n+        return copy\n+\n+    def __getitem__(self, key):\n+        return self.spacy_doc.__getitem__(key)\n+\n+    def __len__(self):\n+        return self.spacy_doc.__len__()\n+\n+    def _impose_token_boundaries(self, boundaries):\n+        b_idx = 0\n+        imposed_tokens = []\n+        for t in self.spacy_doc:\n+            start = t.idx\n+            end = start + len(t)\n+            splits = [0]\n+            while b_idx < len(boundaries) and boundaries[b_idx] < end:\n+                if boundaries[b_idx] > start:\n+                    splits.append(boundaries[b_idx] - start)\n+                b_idx += 1\n+            splits.append(end)\n+            result_tokens = []\n+            for i in xrange(len(splits) - 1):\n+                result_tokens.append([t.text[splits[i]: splits[i+1]], False])\n+            result_tokens[-1][1] = len(t.whitespace_) > 0\n+            imposed_tokens += [tuple(token) for token in result_tokens]\n+        self.spacy_doc = spacy.tokens.Doc(self.spacy_doc.vocab, orths_and_spaces=imposed_tokens)\n+\n+    def _get_entity_boundaries_for_tokenization(self):\n+        spans = set(\n+            itertools.chain.from_iterable(\n+                [e.spans for e in self.brat_annotation.get_entities()]))\n+        return sorted(set(itertools.chain.from_iterable(spans)))\n+\n+    def _regenerate_from_brat(self):\n+        self.entities = [Entity(e, self) for e in self.brat_annotation.get_entities()]\n+        self.relations = [Relation(r, self) for r in self.brat_annotation.get_relations()]\n+\n+    def get_entity(self, id):\n+        return next((e for e in self.entities if e.id == id), None)\n+\n+    def remove_entity(self, entity):\n+        # TODO: make this not destructive\n+        self.entities = [e for e in self.entities if e != entity]\n+        self.brat_annotation.del_annotation(entity.brat_annotation)\n+\n+\n+class Entity:\n+    \"\"\"Wrapper for brat annotation. Spans are in tokens.\"\"\"\n+    def __init__(self, brat_annotation, parent_doc):\n+        self.brat_annotation = brat_annotation\n+        self.parent_doc = parent_doc\n+        self.id = brat_annotation.id\n+        self.type = brat_annotation.type\n+        self.character_spans = brat_annotation.spans\n+        self.spans = []\n+        for span in brat_annotation.spans:\n+            self.spans.append((get_token_starting_at_char_offset(parent_doc, span[0]).i,\n+                               get_token_ending_at_char_offset(parent_doc, span[1]).i + 1))\n+\n+    def same_span(self, other):\n+        return set(self.character_spans) == set(other.character_spans)\n+\n+    def overlaps(self, other):\n+        return any_overlapping_spans(self.brat_annotation, other.brat_annotation)\n+\n+\n+class Relation:\n+    def __init__(self, brat_annotation, parent_doc):\n+        self.brat_annotation = brat_annotation\n+        self.type = brat_annotation.type\n+        self.arg1 = parent_doc.get_entity(brat_annotation.arg1)\n+        self.arg2 = parent_doc.get_entity(brat_annotation.arg2)\n+\n+\n+def get_docs(*paths):\n+    found_paths = []\n+    extensions = re.compile(r\"\\.(txt|ann)$\")\n+    for path in paths:\n+        if os.path.isdir(path):\n+            # Don't recur, just check one level down\n+            children = [f for f in glob.glob(os.path.join(path, \"*\")) if extensions.search(f)]\n+            if not children:\n+                sys.stderr.write(\"No annotation files found in {}\\n\".format(path))\n+            identifiers = [child[:-4] for child in children]\n+            found_paths += list(set(identifiers))\n+        else:\n+            if extensions.search(path):\n+                found_paths.append(path[:-4])\n+            else:\n+                found_paths.append(path)\n+\n+    result = [load_doc(p) for p in found_paths]\n+    return result\n+\n+\n+def load_doc(identifier):\n+    \"\"\"An identifier is the path to a pair of a .ann file and a .txt file, minus\n+    the extension for either.\n+\n+    \"\"\"\n+    return EnhancedAnnotatedDoc(annotation.TextAnnotations(identifier))\n+\n+\n+def get_token_starting_at_char_offset(doc, offset):\n+    l = 0\n+    r = len(doc)\n+    while r - l > 1:\n+        mid = (l + r) / 2\n+        if doc[mid].idx > offset:\n+            r = mid\n+        else:\n+            l = mid\n+\n+    if doc[l].idx == offset:\n+        return doc[l]\n+    else:\n+        return None\n+\n+\n+def get_token_ending_at_char_offset(doc, offset):\n+    l = 0\n+    r = len(doc)\n+    while r - l > 1:\n+        mid = (l + r) / 2\n+        if doc[mid].idx + len(doc[mid]) > offset:\n+            r = mid\n+        else:\n+            l = mid\n+\n+    if doc[l].idx + len(doc[l]) == offset:\n+        return doc[l]\n+    else:\n+        return None\n+\n+\n+def find_overlapping(doc):\n+    return filter(lambda c: any_overlapping_spans(c[0], c[1]),\n+                  itertools.combinations(doc.get_entities(), 2))\n+\n+\n+def any_overlapping_spans(a, b):\n+        for i in a.spans:\n+            for j in b.spans:\n+                if j[0] < i[1] and i[0] < j[1]:\n+                    return True\n+        return False\n+\n+\n+# Requires the spans be independent\n+def spans_to_biluo(spans, total_len):\n+    # o -> 0, b -> 1, i -> 2, l -> 3, u -> 4\n+    spans = sorted(spans)\n+    result = [0] * total_len\n+    i = 0\n+    j = 0\n+    while j < len(spans):\n+        if i >= total_len:\n+            print spans\n+        if i < spans[j][0]:  # o\n+            result[i] = 0\n+        elif i == spans[j][0] and (i + 1 < spans[j][1]):  # b\n+            result[i] = 1\n+        elif i == spans[j][0]:  # u\n+            result[i] = 4\n+            j += 1\n+        elif i + 1 == spans[j][1]:  # l\n+            result[i] = 3\n+            j += 1\n+        else:  # i\n+            result[i] = 2\n+        i += 1\n+    return result\n+\n+\n+def compatibilize_tokenization(original_docs):\n+    boundaries = sorted(set(itertools.chain.from_iterable((\n+        doc._get_entity_boundaries_for_tokenization() for doc in original_docs))))","path":"tools/ai2_common.py","position":214,"original_position":214,"commit_id":"d48e2d37c94ce4004ab91fb5e4c71c64a2e05837","original_commit_id":"d48e2d37c94ce4004ab91fb5e4c71c64a2e05837","user":{"login":"dandersonw","id":14693248,"avatar_url":"https://avatars3.githubusercontent.com/u/14693248?v=3","gravatar_id":"","url":"https://api.github.com/users/dandersonw","html_url":"https://github.com/dandersonw","followers_url":"https://api.github.com/users/dandersonw/followers","following_url":"https://api.github.com/users/dandersonw/following{/other_user}","gists_url":"https://api.github.com/users/dandersonw/gists{/gist_id}","starred_url":"https://api.github.com/users/dandersonw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dandersonw/subscriptions","organizations_url":"https://api.github.com/users/dandersonw/orgs","repos_url":"https://api.github.com/users/dandersonw/repos","events_url":"https://api.github.com/users/dandersonw/events{/privacy}","received_events_url":"https://api.github.com/users/dandersonw/received_events","type":"User","site_admin":false},"body":"I will make the change as you suggest - it will indeed make life easier. I was operating under what I thought was an earlier discussion that we wanted to respect annotation boundaries despite what spacy would give us for tokenization.","created_at":"2017-05-26T20:46:50Z","updated_at":"2017-05-26T20:46:50Z","html_url":"https://github.com/allenai/brat/pull/5#discussion_r118787001","pull_request_url":"https://api.github.com/repos/allenai/brat/pulls/5","_links":{"self":{"href":"https://api.github.com/repos/allenai/brat/pulls/comments/118787001"},"html":{"href":"https://github.com/allenai/brat/pull/5#discussion_r118787001"},"pull_request":{"href":"https://api.github.com/repos/allenai/brat/pulls/5"}}},"pull_request":{"url":"https://api.github.com/repos/allenai/brat/pulls/5","id":122344327,"html_url":"https://github.com/allenai/brat/pull/5","diff_url":"https://github.com/allenai/brat/pull/5.diff","patch_url":"https://github.com/allenai/brat/pull/5.patch","issue_url":"https://api.github.com/repos/allenai/brat/issues/5","number":5,"state":"open","locked":false,"title":"Tooling for working with annotations","user":{"login":"dandersonw","id":14693248,"avatar_url":"https://avatars3.githubusercontent.com/u/14693248?v=3","gravatar_id":"","url":"https://api.github.com/users/dandersonw","html_url":"https://github.com/dandersonw","followers_url":"https://api.github.com/users/dandersonw/followers","following_url":"https://api.github.com/users/dandersonw/following{/other_user}","gists_url":"https://api.github.com/users/dandersonw/gists{/gist_id}","starred_url":"https://api.github.com/users/dandersonw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dandersonw/subscriptions","organizations_url":"https://api.github.com/users/dandersonw/orgs","repos_url":"https://api.github.com/users/dandersonw/repos","events_url":"https://api.github.com/users/dandersonw/events{/privacy}","received_events_url":"https://api.github.com/users/dandersonw/received_events","type":"User","site_admin":false},"body":"Not yet ready to merge","created_at":"2017-05-24T23:27:37Z","updated_at":"2017-05-26T20:46:50Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a6ca37dbe619985b27a349b3aa969fb7e0bbb4da","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/allenai/brat/pulls/5/commits","review_comments_url":"https://api.github.com/repos/allenai/brat/pulls/5/comments","review_comment_url":"https://api.github.com/repos/allenai/brat/pulls/comments{/number}","comments_url":"https://api.github.com/repos/allenai/brat/issues/5/comments","statuses_url":"https://api.github.com/repos/allenai/brat/statuses/d48e2d37c94ce4004ab91fb5e4c71c64a2e05837","head":{"label":"dandersonw:measure-agreement","ref":"measure-agreement","sha":"d48e2d37c94ce4004ab91fb5e4c71c64a2e05837","user":{"login":"dandersonw","id":14693248,"avatar_url":"https://avatars3.githubusercontent.com/u/14693248?v=3","gravatar_id":"","url":"https://api.github.com/users/dandersonw","html_url":"https://github.com/dandersonw","followers_url":"https://api.github.com/users/dandersonw/followers","following_url":"https://api.github.com/users/dandersonw/following{/other_user}","gists_url":"https://api.github.com/users/dandersonw/gists{/gist_id}","starred_url":"https://api.github.com/users/dandersonw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dandersonw/subscriptions","organizations_url":"https://api.github.com/users/dandersonw/orgs","repos_url":"https://api.github.com/users/dandersonw/repos","events_url":"https://api.github.com/users/dandersonw/events{/privacy}","received_events_url":"https://api.github.com/users/dandersonw/received_events","type":"User","site_admin":false},"repo":{"id":91706730,"name":"brat","full_name":"dandersonw/brat","owner":{"login":"dandersonw","id":14693248,"avatar_url":"https://avatars3.githubusercontent.com/u/14693248?v=3","gravatar_id":"","url":"https://api.github.com/users/dandersonw","html_url":"https://github.com/dandersonw","followers_url":"https://api.github.com/users/dandersonw/followers","following_url":"https://api.github.com/users/dandersonw/following{/other_user}","gists_url":"https://api.github.com/users/dandersonw/gists{/gist_id}","starred_url":"https://api.github.com/users/dandersonw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dandersonw/subscriptions","organizations_url":"https://api.github.com/users/dandersonw/orgs","repos_url":"https://api.github.com/users/dandersonw/repos","events_url":"https://api.github.com/users/dandersonw/events{/privacy}","received_events_url":"https://api.github.com/users/dandersonw/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/dandersonw/brat","description":"brat rapid annotation tool (brat) - for all your textual annotation needs","fork":true,"url":"https://api.github.com/repos/dandersonw/brat","forks_url":"https://api.github.com/repos/dandersonw/brat/forks","keys_url":"https://api.github.com/repos/dandersonw/brat/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dandersonw/brat/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dandersonw/brat/teams","hooks_url":"https://api.github.com/repos/dandersonw/brat/hooks","issue_events_url":"https://api.github.com/repos/dandersonw/brat/issues/events{/number}","events_url":"https://api.github.com/repos/dandersonw/brat/events","assignees_url":"https://api.github.com/repos/dandersonw/brat/assignees{/user}","branches_url":"https://api.github.com/repos/dandersonw/brat/branches{/branch}","tags_url":"https://api.github.com/repos/dandersonw/brat/tags","blobs_url":"https://api.github.com/repos/dandersonw/brat/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dandersonw/brat/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dandersonw/brat/git/refs{/sha}","trees_url":"https://api.github.com/repos/dandersonw/brat/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dandersonw/brat/statuses/{sha}","languages_url":"https://api.github.com/repos/dandersonw/brat/languages","stargazers_url":"https://api.github.com/repos/dandersonw/brat/stargazers","contributors_url":"https://api.github.com/repos/dandersonw/brat/contributors","subscribers_url":"https://api.github.com/repos/dandersonw/brat/subscribers","subscription_url":"https://api.github.com/repos/dandersonw/brat/subscription","commits_url":"https://api.github.com/repos/dandersonw/brat/commits{/sha}","git_commits_url":"https://api.github.com/repos/dandersonw/brat/git/commits{/sha}","comments_url":"https://api.github.com/repos/dandersonw/brat/comments{/number}","issue_comment_url":"https://api.github.com/repos/dandersonw/brat/issues/comments{/number}","contents_url":"https://api.github.com/repos/dandersonw/brat/contents/{+path}","compare_url":"https://api.github.com/repos/dandersonw/brat/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dandersonw/brat/merges","archive_url":"https://api.github.com/repos/dandersonw/brat/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dandersonw/brat/downloads","issues_url":"https://api.github.com/repos/dandersonw/brat/issues{/number}","pulls_url":"https://api.github.com/repos/dandersonw/brat/pulls{/number}","milestones_url":"https://api.github.com/repos/dandersonw/brat/milestones{/number}","notifications_url":"https://api.github.com/repos/dandersonw/brat/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dandersonw/brat/labels{/name}","releases_url":"https://api.github.com/repos/dandersonw/brat/releases{/id}","deployments_url":"https://api.github.com/repos/dandersonw/brat/deployments","created_at":"2017-05-18T15:00:55Z","updated_at":"2017-05-18T15:01:16Z","pushed_at":"2017-05-26T18:59:29Z","git_url":"git://github.com/dandersonw/brat.git","ssh_url":"git@github.com:dandersonw/brat.git","clone_url":"https://github.com/dandersonw/brat.git","svn_url":"https://github.com/dandersonw/brat","homepage":"http://brat.nlplab.org","size":102962,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"allenai:master","ref":"master","sha":"ad0b8ad1628d7be86f35290c5e3d81559e04a4c4","user":{"login":"allenai","id":5667695,"avatar_url":"https://avatars0.githubusercontent.com/u/5667695?v=3","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"repo":{"id":86846992,"name":"brat","full_name":"allenai/brat","owner":{"login":"allenai","id":5667695,"avatar_url":"https://avatars0.githubusercontent.com/u/5667695?v=3","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/allenai/brat","description":"brat rapid annotation tool (brat) - for all your textual annotation needs","fork":true,"url":"https://api.github.com/repos/allenai/brat","forks_url":"https://api.github.com/repos/allenai/brat/forks","keys_url":"https://api.github.com/repos/allenai/brat/keys{/key_id}","collaborators_url":"https://api.github.com/repos/allenai/brat/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/allenai/brat/teams","hooks_url":"https://api.github.com/repos/allenai/brat/hooks","issue_events_url":"https://api.github.com/repos/allenai/brat/issues/events{/number}","events_url":"https://api.github.com/repos/allenai/brat/events","assignees_url":"https://api.github.com/repos/allenai/brat/assignees{/user}","branches_url":"https://api.github.com/repos/allenai/brat/branches{/branch}","tags_url":"https://api.github.com/repos/allenai/brat/tags","blobs_url":"https://api.github.com/repos/allenai/brat/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/allenai/brat/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/allenai/brat/git/refs{/sha}","trees_url":"https://api.github.com/repos/allenai/brat/git/trees{/sha}","statuses_url":"https://api.github.com/repos/allenai/brat/statuses/{sha}","languages_url":"https://api.github.com/repos/allenai/brat/languages","stargazers_url":"https://api.github.com/repos/allenai/brat/stargazers","contributors_url":"https://api.github.com/repos/allenai/brat/contributors","subscribers_url":"https://api.github.com/repos/allenai/brat/subscribers","subscription_url":"https://api.github.com/repos/allenai/brat/subscription","commits_url":"https://api.github.com/repos/allenai/brat/commits{/sha}","git_commits_url":"https://api.github.com/repos/allenai/brat/git/commits{/sha}","comments_url":"https://api.github.com/repos/allenai/brat/comments{/number}","issue_comment_url":"https://api.github.com/repos/allenai/brat/issues/comments{/number}","contents_url":"https://api.github.com/repos/allenai/brat/contents/{+path}","compare_url":"https://api.github.com/repos/allenai/brat/compare/{base}...{head}","merges_url":"https://api.github.com/repos/allenai/brat/merges","archive_url":"https://api.github.com/repos/allenai/brat/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/allenai/brat/downloads","issues_url":"https://api.github.com/repos/allenai/brat/issues{/number}","pulls_url":"https://api.github.com/repos/allenai/brat/pulls{/number}","milestones_url":"https://api.github.com/repos/allenai/brat/milestones{/number}","notifications_url":"https://api.github.com/repos/allenai/brat/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/allenai/brat/labels{/name}","releases_url":"https://api.github.com/repos/allenai/brat/releases{/id}","deployments_url":"https://api.github.com/repos/allenai/brat/deployments","created_at":"2017-03-31T18:04:31Z","updated_at":"2017-05-25T10:25:08Z","pushed_at":"2017-05-26T20:25:23Z","git_url":"git://github.com/allenai/brat.git","ssh_url":"git@github.com:allenai/brat.git","clone_url":"https://github.com/allenai/brat.git","svn_url":"https://github.com/allenai/brat","homepage":"http://brat.nlplab.org","size":102913,"stargazers_count":3,"watchers_count":3,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/allenai/brat/pulls/5"},"html":{"href":"https://github.com/allenai/brat/pull/5"},"issue":{"href":"https://api.github.com/repos/allenai/brat/issues/5"},"comments":{"href":"https://api.github.com/repos/allenai/brat/issues/5/comments"},"review_comments":{"href":"https://api.github.com/repos/allenai/brat/pulls/5/comments"},"review_comment":{"href":"https://api.github.com/repos/allenai/brat/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/allenai/brat/pulls/5/commits"},"statuses":{"href":"https://api.github.com/repos/allenai/brat/statuses/d48e2d37c94ce4004ab91fb5e4c71c64a2e05837"}}}}
{"actor":{"display_login":"dandersonw"}}