{"action":"created","comment":{"url":"https://api.github.com/repos/allenai/allennlp/pulls/comments/142301760","pull_request_review_id":66642694,"id":142301760,"diff_hunk":"@@ -0,0 +1,244 @@\n+import logging\n+import math\n+from typing import Any, Dict, List, Optional\n+\n+import torch\n+import torch.nn.functional as F\n+\n+from allennlp.common import Params\n+from allennlp.data import Vocabulary\n+from allennlp.models.model import Model\n+from allennlp.modules.token_embedders import Embedding\n+from allennlp.modules import FeedForward\n+from allennlp.modules import Seq2SeqEncoder, TimeDistributed, TextFieldEmbedder\n+from allennlp.nn import util, InitializerApplicator, RegularizerApplicator\n+from allennlp.training.metrics import MentionRecall, ConllScores\n+\n+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name\n+\n+@Model.register(\"coref\")\n+class CoreferenceResolver(Model):\n+    def __init__(self, vocab: Vocabulary,\n+                 text_field_embedder: TextFieldEmbedder,\n+                 context_layer: Seq2SeqEncoder,\n+                 mention_feedforward: FeedForward,\n+                 antecedent_feedforward: FeedForward,\n+                 feature_size: int,\n+                 max_span_width: int,\n+                 spans_per_word: float,\n+                 max_antecedents: int,\n+                 lexical_dropout: float = 0.2,\n+                 initializer: InitializerApplicator = InitializerApplicator(),\n+                 regularizer: Optional[RegularizerApplicator] = None) -> None:\n+        super(CoreferenceResolver, self).__init__(vocab, regularizer)\n+        self._text_field_embedder = text_field_embedder\n+        self._context_layer = context_layer\n+        self._mention_feedforward = TimeDistributed(mention_feedforward)\n+        self._antecedent_feedforward = TimeDistributed(antecedent_feedforward)\n+        self._mention_scorer = TimeDistributed(torch.nn.Linear(mention_feedforward.get_output_dim(), 1))\n+        self._antecedent_scorer = TimeDistributed(torch.nn.Linear(antecedent_feedforward.get_output_dim(), 1))\n+        self._head_scorer = TimeDistributed(torch.nn.Linear(context_layer.get_output_dim(), 1))\n+        self._distance_embedding = Embedding(10, feature_size) # 10 possible distance buckets.\n+        self._span_width_embedding = Embedding(max_span_width, feature_size)\n+        self._max_span_width = max_span_width\n+        self._spans_per_word = spans_per_word\n+        self._max_antecedents = max_antecedents\n+        self._mention_recall = MentionRecall()\n+        self._conll_scores = ConllScores()\n+        if lexical_dropout > 0:\n+            self._lexical_dropout = torch.nn.Dropout(p=lexical_dropout)\n+        else:\n+            self._lexical_dropout = lambda x: x\n+        initializer(self)\n+\n+    @staticmethod\n+    def _bucket_distance(distances):\n+        # Buckets: [0, 1, 2, 3, 4, 5-7, 8-15, 16-31, 32-63, 64+].\n+        logspace_idx = (distances.float().log()/math.log(2)).floor().long() + 3\n+        use_identity = (distances <= 4).long()\n+        combined_idx = use_identity * distances + (1 + (-1 * use_identity)) * logspace_idx\n+        return combined_idx.clamp(0, 9)\n+\n+    def forward(self,  # type: ignore\n+                text: Dict[str, torch.LongTensor],\n+                span_starts: torch.IntTensor,\n+                span_ends: torch.IntTensor,\n+                span_labels: torch.IntTensor,\n+                metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]:\n+        # pylint: disable=arguments-differ\n+        # Basic setup.\n+        text_emb = self._lexical_dropout(self._text_field_embedder(text)) # [b, text_len, emb]\n+        is_cuda = text_emb.is_cuda\n+        batch_size = span_labels.size(0)\n+        num_spans = span_labels.size(1)\n+        text_len = text_emb.size(1)\n+        text_mask = util.get_text_field_mask(text).float() # [b, text_len]\n+        span_mask = (span_starts >= 0).float() # [b, num_spans, 1]\n+        span_starts = F.relu(span_starts.float()).long() # [b, num_spans, 1]\n+        span_ends = F.relu(span_ends.float()).long() # [b, num_spans, 1]\n+\n+        # Compute span representations.\n+        contextualized_emb = self._context_layer(text_emb, text_mask) # [b, text_len, emb]\n+        start_emb = util.batched_index_select(contextualized_emb,\n+                                              span_starts.squeeze(-1)) # [b, num_spans, emb]\n+        end_emb = util.batched_index_select(contextualized_emb,\n+                                            span_ends.squeeze(-1)) # [b, num_spans, emb]\n+        span_size = span_ends - span_starts # [b, num_spans, 1] (really span size minus one)\n+        span_size_emb = self._span_width_embedding(span_size.squeeze(-1)) # [b, num_spans, emb]\n+        head_scores = self._head_scorer(contextualized_emb) # [b, text_len, 1]\n+        head_offsets = util.get_indices(self._max_span_width, is_cuda).view(1, 1, -1) # [1, 1, max_span_width]\n+        head_mask = (head_offsets <= span_size).float() # [b, num_spans, max_span_width]\n+        raw_head_idx = span_ends - head_offsets # [b, num_spans, max_span_width]\n+        head_mask = head_mask * (raw_head_idx >= 0).float() # [b, num_spans, max_span_width]\n+        head_idx = F.relu(raw_head_idx.float()).long() # [b, num_spans, max_span_width]\n+        flat_head_idx = util.flatten_batched_indices(head_idx,\n+                                                     text_len) # [b * num_spans * max_span_width]\n+        span_text_emb = util.batched_index_select(text_emb,\n+                                                  head_idx,\n+                                                  flat_head_idx) # [b, num_spans, max_span_width, emb]\n+        span_head_scores = util.batched_index_select(head_scores,\n+                                                     head_idx,\n+                                                     flat_head_idx).squeeze(-1) # [b, num_spans, max_span_width]\n+        span_head_scores += head_mask.float().log() # [b, num_spans, max_span_width]","path":"allennlp/models/coreference_resolution/coref.py","position":102,"original_position":102,"commit_id":"f204e7485e951f8ebbc439224aed2e63cc31771b","original_commit_id":"f204e7485e951f8ebbc439224aed2e63cc31771b","user":{"login":"kentonl","id":2646947,"avatar_url":"https://avatars3.githubusercontent.com/u/2646947?v=4","gravatar_id":"","url":"https://api.github.com/users/kentonl","html_url":"https://github.com/kentonl","followers_url":"https://api.github.com/users/kentonl/followers","following_url":"https://api.github.com/users/kentonl/following{/other_user}","gists_url":"https://api.github.com/users/kentonl/gists{/gist_id}","starred_url":"https://api.github.com/users/kentonl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kentonl/subscriptions","organizations_url":"https://api.github.com/users/kentonl/orgs","repos_url":"https://api.github.com/users/kentonl/repos","events_url":"https://api.github.com/users/kentonl/events{/privacy}","received_events_url":"https://api.github.com/users/kentonl/received_events","type":"User","site_admin":false},"body":"IMO, this is a lot cleaner and faster than what's in masked_softmax. Masking in logspace only requires a single elementwise multiply. In practice there are no numerical stability issues. I was actually going to add a pull request that would fix this in bidaf...","created_at":"2017-10-03T02:44:12Z","updated_at":"2017-10-03T02:44:12Z","html_url":"https://github.com/allenai/allennlp/pull/364#discussion_r142301760","pull_request_url":"https://api.github.com/repos/allenai/allennlp/pulls/364","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/comments/142301760"},"html":{"href":"https://github.com/allenai/allennlp/pull/364#discussion_r142301760"},"pull_request":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/364"}},"in_reply_to_id":142284448},"pull_request":{"url":"https://api.github.com/repos/allenai/allennlp/pulls/364","id":144102754,"html_url":"https://github.com/allenai/allennlp/pull/364","diff_url":"https://github.com/allenai/allennlp/pull/364.diff","patch_url":"https://github.com/allenai/allennlp/pull/364.patch","issue_url":"https://api.github.com/repos/allenai/allennlp/issues/364","number":364,"state":"open","locked":false,"title":"Initial simplified e2e-coref implementation.","user":{"login":"kentonl","id":2646947,"avatar_url":"https://avatars3.githubusercontent.com/u/2646947?v=4","gravatar_id":"","url":"https://api.github.com/users/kentonl","html_url":"https://github.com/kentonl","followers_url":"https://api.github.com/users/kentonl/followers","following_url":"https://api.github.com/users/kentonl/following{/other_user}","gists_url":"https://api.github.com/users/kentonl/gists{/gist_id}","starred_url":"https://api.github.com/users/kentonl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kentonl/subscriptions","organizations_url":"https://api.github.com/users/kentonl/orgs","repos_url":"https://api.github.com/users/kentonl/repos","events_url":"https://api.github.com/users/kentonl/events{/privacy}","received_events_url":"https://api.github.com/users/kentonl/received_events","type":"User","site_admin":false},"body":"Here is a slightly simplified implementation of the coreference resolution system from here: https://arxiv.org/abs/1707.07045. This version achieves ~63 dev F1, compared to the reported ~67 dev F1.\r\n\r\nThere are a few differences that make the results worse than in the paper.\r\n(1) This only considers up to 100 antecedents (rather than 250) due to memory constraints. The original implementation dynamically and randomly generated slices of documents that are too long. I'm not sure if that is currently supported by AllenNLP.\r\n(2) This does not apply recurrent variational dropout.\r\n(3) Does not enforce the non-crossing constraint between predicted mentions. This probably requires writing a custom kernel to do the greedy pruning. It's also not important unless you need to use the official CoNLL evaluation scripts, which implicitly require this constraint due to their formatting.\r\n(4) Does not use genre and speaker metadata (unrealistic in many applications anyways...).\r\n(5) Does not ensemble the models.","created_at":"2017-10-02T02:42:01Z","updated_at":"2017-10-03T02:44:12Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6593dfbb1e3ac99c85a4830a2edbe97ee5fd74e3","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/allenai/allennlp/pulls/364/commits","review_comments_url":"https://api.github.com/repos/allenai/allennlp/pulls/364/comments","review_comment_url":"https://api.github.com/repos/allenai/allennlp/pulls/comments{/number}","comments_url":"https://api.github.com/repos/allenai/allennlp/issues/364/comments","statuses_url":"https://api.github.com/repos/allenai/allennlp/statuses/f204e7485e951f8ebbc439224aed2e63cc31771b","head":{"label":"kentonl:coref","ref":"coref","sha":"f204e7485e951f8ebbc439224aed2e63cc31771b","user":{"login":"kentonl","id":2646947,"avatar_url":"https://avatars3.githubusercontent.com/u/2646947?v=4","gravatar_id":"","url":"https://api.github.com/users/kentonl","html_url":"https://github.com/kentonl","followers_url":"https://api.github.com/users/kentonl/followers","following_url":"https://api.github.com/users/kentonl/following{/other_user}","gists_url":"https://api.github.com/users/kentonl/gists{/gist_id}","starred_url":"https://api.github.com/users/kentonl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kentonl/subscriptions","organizations_url":"https://api.github.com/users/kentonl/orgs","repos_url":"https://api.github.com/users/kentonl/repos","events_url":"https://api.github.com/users/kentonl/events{/privacy}","received_events_url":"https://api.github.com/users/kentonl/received_events","type":"User","site_admin":false},"repo":{"id":105305747,"name":"allennlp","full_name":"kentonl/allennlp","owner":{"login":"kentonl","id":2646947,"avatar_url":"https://avatars3.githubusercontent.com/u/2646947?v=4","gravatar_id":"","url":"https://api.github.com/users/kentonl","html_url":"https://github.com/kentonl","followers_url":"https://api.github.com/users/kentonl/followers","following_url":"https://api.github.com/users/kentonl/following{/other_user}","gists_url":"https://api.github.com/users/kentonl/gists{/gist_id}","starred_url":"https://api.github.com/users/kentonl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kentonl/subscriptions","organizations_url":"https://api.github.com/users/kentonl/orgs","repos_url":"https://api.github.com/users/kentonl/repos","events_url":"https://api.github.com/users/kentonl/events{/privacy}","received_events_url":"https://api.github.com/users/kentonl/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kentonl/allennlp","description":"An open-source NLP research library, built on PyTorch.","fork":true,"url":"https://api.github.com/repos/kentonl/allennlp","forks_url":"https://api.github.com/repos/kentonl/allennlp/forks","keys_url":"https://api.github.com/repos/kentonl/allennlp/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kentonl/allennlp/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kentonl/allennlp/teams","hooks_url":"https://api.github.com/repos/kentonl/allennlp/hooks","issue_events_url":"https://api.github.com/repos/kentonl/allennlp/issues/events{/number}","events_url":"https://api.github.com/repos/kentonl/allennlp/events","assignees_url":"https://api.github.com/repos/kentonl/allennlp/assignees{/user}","branches_url":"https://api.github.com/repos/kentonl/allennlp/branches{/branch}","tags_url":"https://api.github.com/repos/kentonl/allennlp/tags","blobs_url":"https://api.github.com/repos/kentonl/allennlp/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kentonl/allennlp/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kentonl/allennlp/git/refs{/sha}","trees_url":"https://api.github.com/repos/kentonl/allennlp/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kentonl/allennlp/statuses/{sha}","languages_url":"https://api.github.com/repos/kentonl/allennlp/languages","stargazers_url":"https://api.github.com/repos/kentonl/allennlp/stargazers","contributors_url":"https://api.github.com/repos/kentonl/allennlp/contributors","subscribers_url":"https://api.github.com/repos/kentonl/allennlp/subscribers","subscription_url":"https://api.github.com/repos/kentonl/allennlp/subscription","commits_url":"https://api.github.com/repos/kentonl/allennlp/commits{/sha}","git_commits_url":"https://api.github.com/repos/kentonl/allennlp/git/commits{/sha}","comments_url":"https://api.github.com/repos/kentonl/allennlp/comments{/number}","issue_comment_url":"https://api.github.com/repos/kentonl/allennlp/issues/comments{/number}","contents_url":"https://api.github.com/repos/kentonl/allennlp/contents/{+path}","compare_url":"https://api.github.com/repos/kentonl/allennlp/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kentonl/allennlp/merges","archive_url":"https://api.github.com/repos/kentonl/allennlp/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kentonl/allennlp/downloads","issues_url":"https://api.github.com/repos/kentonl/allennlp/issues{/number}","pulls_url":"https://api.github.com/repos/kentonl/allennlp/pulls{/number}","milestones_url":"https://api.github.com/repos/kentonl/allennlp/milestones{/number}","notifications_url":"https://api.github.com/repos/kentonl/allennlp/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kentonl/allennlp/labels{/name}","releases_url":"https://api.github.com/repos/kentonl/allennlp/releases{/id}","deployments_url":"https://api.github.com/repos/kentonl/allennlp/deployments","created_at":"2017-09-29T18:41:13Z","updated_at":"2017-09-29T18:41:15Z","pushed_at":"2017-10-02T05:58:03Z","git_url":"git://github.com/kentonl/allennlp.git","ssh_url":"git@github.com:kentonl/allennlp.git","clone_url":"https://github.com/kentonl/allennlp.git","svn_url":"https://github.com/kentonl/allennlp","homepage":"http://www.allennlp.org/","size":5772,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"allenai:master","ref":"master","sha":"a70849dc8e0ade58f6580110f9032e2adea42b4d","user":{"login":"allenai","id":5667695,"avatar_url":"https://avatars3.githubusercontent.com/u/5667695?v=4","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"repo":{"id":91356408,"name":"allennlp","full_name":"allenai/allennlp","owner":{"login":"allenai","id":5667695,"avatar_url":"https://avatars3.githubusercontent.com/u/5667695?v=4","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/allenai/allennlp","description":"An open-source NLP research library, built on PyTorch.","fork":false,"url":"https://api.github.com/repos/allenai/allennlp","forks_url":"https://api.github.com/repos/allenai/allennlp/forks","keys_url":"https://api.github.com/repos/allenai/allennlp/keys{/key_id}","collaborators_url":"https://api.github.com/repos/allenai/allennlp/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/allenai/allennlp/teams","hooks_url":"https://api.github.com/repos/allenai/allennlp/hooks","issue_events_url":"https://api.github.com/repos/allenai/allennlp/issues/events{/number}","events_url":"https://api.github.com/repos/allenai/allennlp/events","assignees_url":"https://api.github.com/repos/allenai/allennlp/assignees{/user}","branches_url":"https://api.github.com/repos/allenai/allennlp/branches{/branch}","tags_url":"https://api.github.com/repos/allenai/allennlp/tags","blobs_url":"https://api.github.com/repos/allenai/allennlp/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/allenai/allennlp/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/allenai/allennlp/git/refs{/sha}","trees_url":"https://api.github.com/repos/allenai/allennlp/git/trees{/sha}","statuses_url":"https://api.github.com/repos/allenai/allennlp/statuses/{sha}","languages_url":"https://api.github.com/repos/allenai/allennlp/languages","stargazers_url":"https://api.github.com/repos/allenai/allennlp/stargazers","contributors_url":"https://api.github.com/repos/allenai/allennlp/contributors","subscribers_url":"https://api.github.com/repos/allenai/allennlp/subscribers","subscription_url":"https://api.github.com/repos/allenai/allennlp/subscription","commits_url":"https://api.github.com/repos/allenai/allennlp/commits{/sha}","git_commits_url":"https://api.github.com/repos/allenai/allennlp/git/commits{/sha}","comments_url":"https://api.github.com/repos/allenai/allennlp/comments{/number}","issue_comment_url":"https://api.github.com/repos/allenai/allennlp/issues/comments{/number}","contents_url":"https://api.github.com/repos/allenai/allennlp/contents/{+path}","compare_url":"https://api.github.com/repos/allenai/allennlp/compare/{base}...{head}","merges_url":"https://api.github.com/repos/allenai/allennlp/merges","archive_url":"https://api.github.com/repos/allenai/allennlp/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/allenai/allennlp/downloads","issues_url":"https://api.github.com/repos/allenai/allennlp/issues{/number}","pulls_url":"https://api.github.com/repos/allenai/allennlp/pulls{/number}","milestones_url":"https://api.github.com/repos/allenai/allennlp/milestones{/number}","notifications_url":"https://api.github.com/repos/allenai/allennlp/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/allenai/allennlp/labels{/name}","releases_url":"https://api.github.com/repos/allenai/allennlp/releases{/id}","deployments_url":"https://api.github.com/repos/allenai/allennlp/deployments","created_at":"2017-05-15T15:52:41Z","updated_at":"2017-10-03T01:52:26Z","pushed_at":"2017-10-02T22:40:35Z","git_url":"git://github.com/allenai/allennlp.git","ssh_url":"git@github.com:allenai/allennlp.git","clone_url":"https://github.com/allenai/allennlp.git","svn_url":"https://github.com/allenai/allennlp","homepage":"http://www.allennlp.org/","size":5765,"stargazers_count":913,"watchers_count":913,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":79,"mirror_url":null,"open_issues_count":27,"forks":79,"open_issues":27,"watchers":913,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/364"},"html":{"href":"https://github.com/allenai/allennlp/pull/364"},"issue":{"href":"https://api.github.com/repos/allenai/allennlp/issues/364"},"comments":{"href":"https://api.github.com/repos/allenai/allennlp/issues/364/comments"},"review_comments":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/364/comments"},"review_comment":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/364/commits"},"statuses":{"href":"https://api.github.com/repos/allenai/allennlp/statuses/f204e7485e951f8ebbc439224aed2e63cc31771b"}},"author_association":"NONE"}}
{"actor":{"display_login":"kentonl"}}