{"action":"created","comment":{"url":"https://api.github.com/repos/castorini/Anserini/pulls/comments/116362293","pull_request_review_id":37983180,"id":116362293,"diff_hunk":"@@ -0,0 +1,142 @@\n+/**\n+ * Anserini: An information retrieval toolkit built on Lucene\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package io.anserini.qa.passage;\n+\n+import com.google.common.collect.MinMaxPriorityQueue;\n+import io.anserini.embeddings.WordEmbeddingDictionary;\n+import org.apache.lucene.analysis.TokenStream;\n+import org.apache.lucene.analysis.standard.StandardAnalyzer;\n+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;\n+import org.json.JSONObject;\n+\n+import java.io.*;\n+import java.util.*;\n+\n+public class WmdPassageScorer implements PassageScorer {\n+\n+  private final WordEmbeddingDictionary wmdDictionary;\n+  private final MinMaxPriorityQueue<ScoredPassage> scoredPassageHeap;\n+  private final int topPassages;\n+  private final List<String> stopWords;\n+\n+  public WmdPassageScorer(String index, Integer k) throws IOException {\n+    this.wmdDictionary = new WordEmbeddingDictionary(index);\n+    wmdDictionary.getEmbeddingVector(\"newton\");\n+    this.topPassages = k;\n+    scoredPassageHeap = MinMaxPriorityQueue.maximumSize(topPassages).create();\n+    stopWords = new ArrayList<>();\n+\n+    //Get file from resources folder\n+    InputStream is = getClass().getResourceAsStream(\"/io/anserini/qa/english-stoplist.txt\");\n+    BufferedReader bRdr = new BufferedReader(new InputStreamReader(is));\n+    String line;\n+    while ((line = bRdr.readLine()) != null) {\n+      if (!line.contains(\"#\")) {\n+        stopWords.add(line);\n+      }\n+    }\n+  }\n+\n+  public double distance(float[] leftVector, float[] rightVector) {\n+    double sum = 0.0;\n+\n+    for (int i = 0; i < leftVector.length; i++) {\n+      sum += Math.pow(leftVector[i] - rightVector[i], 2);\n+    }\n+    return  Math.sqrt(sum);\n+  }\n+\n+  @Override\n+  public void score(String query, Map<String, Float> sentences) throws Exception {\n+    StandardAnalyzer sa = new StandardAnalyzer();\n+    TokenStream tokenStream = sa.tokenStream(\"contents\", new StringReader(query));\n+    CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);\n+    tokenStream.reset();\n+\n+    HashSet<String> questionTerms = new HashSet<>();\n+    HashSet<String> candidateTerms = new HashSet<>();\n+\n+    // avoid duplicate passages\n+    HashSet<String> seenSentences = new HashSet<>();\n+\n+    while (tokenStream.incrementToken()) {\n+      questionTerms.add(charTermAttribute.toString());\n+    }\n+\n+    for (Map.Entry<String, Float> sent : sentences.entrySet()) {\n+      double wmd = 0.0;\n+      candidateTerms.clear();\n+      StandardAnalyzer sa2 = new StandardAnalyzer();\n+      TokenStream candTokenStream = sa2.tokenStream(\"contents\", new StringReader(sent.getKey()));\n+      charTermAttribute = candTokenStream.addAttribute(CharTermAttribute.class);\n+      candTokenStream.reset();\n+\n+      while (candTokenStream.incrementToken()) {\n+        candidateTerms.add(charTermAttribute.toString());\n+      }\n+\n+      for(String qTerm : questionTerms) {\n+        double minWMD = Double.MAX_VALUE;\n+        for (String candTerm : candidateTerms) {\n+          try {\n+            double thisWMD = distance(wmdDictionary.getEmbeddingVector(qTerm), wmdDictionary.getEmbeddingVector(candTerm));\n+            if (minWMD > thisWMD) {\n+              minWMD = thisWMD;\n+            }\n+          } catch (ArrayIndexOutOfBoundsException e) {\n+            // term is OOV\n+          }\n+        }\n+        if (minWMD != Double.MAX_VALUE)","path":"src/main/java/io/anserini/qa/passage/WmdPassageScorer.java","position":103,"original_position":103,"commit_id":"efb434386a5ffdd85c22e614060edf3d245b3c77","original_commit_id":"efb434386a5ffdd85c22e614060edf3d245b3c77","user":{"login":"minafarid","id":9206118,"avatar_url":"https://avatars3.githubusercontent.com/u/9206118?v=3","gravatar_id":"","url":"https://api.github.com/users/minafarid","html_url":"https://github.com/minafarid","followers_url":"https://api.github.com/users/minafarid/followers","following_url":"https://api.github.com/users/minafarid/following{/other_user}","gists_url":"https://api.github.com/users/minafarid/gists{/gist_id}","starred_url":"https://api.github.com/users/minafarid/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/minafarid/subscriptions","organizations_url":"https://api.github.com/users/minafarid/orgs","repos_url":"https://api.github.com/users/minafarid/repos","events_url":"https://api.github.com/users/minafarid/events{/privacy}","received_events_url":"https://api.github.com/users/minafarid/received_events","type":"User","site_admin":false},"body":"add brackets to the `if` statement, even if it is only one line... be consistent","created_at":"2017-05-13T16:46:05Z","updated_at":"2017-05-13T16:47:54Z","html_url":"https://github.com/castorini/Anserini/pull/185#discussion_r116362293","pull_request_url":"https://api.github.com/repos/castorini/Anserini/pulls/185","_links":{"self":{"href":"https://api.github.com/repos/castorini/Anserini/pulls/comments/116362293"},"html":{"href":"https://github.com/castorini/Anserini/pull/185#discussion_r116362293"},"pull_request":{"href":"https://api.github.com/repos/castorini/Anserini/pulls/185"}}},"pull_request":{"url":"https://api.github.com/repos/castorini/Anserini/pulls/185","id":120433473,"html_url":"https://github.com/castorini/Anserini/pull/185","diff_url":"https://github.com/castorini/Anserini/pull/185.diff","patch_url":"https://github.com/castorini/Anserini/pull/185.patch","issue_url":"https://api.github.com/repos/castorini/Anserini/issues/185","number":185,"state":"open","locked":false,"title":"WMD passage scorer (#142)","user":{"login":"rosequ","id":8274617,"avatar_url":"https://avatars0.githubusercontent.com/u/8274617?v=3","gravatar_id":"","url":"https://api.github.com/users/rosequ","html_url":"https://github.com/rosequ","followers_url":"https://api.github.com/users/rosequ/followers","following_url":"https://api.github.com/users/rosequ/following{/other_user}","gists_url":"https://api.github.com/users/rosequ/gists{/gist_id}","starred_url":"https://api.github.com/users/rosequ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rosequ/subscriptions","organizations_url":"https://api.github.com/users/rosequ/orgs","repos_url":"https://api.github.com/users/rosequ/repos","events_url":"https://api.github.com/users/rosequ/events{/privacy}","received_events_url":"https://api.github.com/users/rosequ/received_events","type":"User","site_admin":false},"body":"- Word-Movers Distance based passage scorer\r\n@nims11 @minafarid","created_at":"2017-05-13T03:38:00Z","updated_at":"2017-05-13T16:47:54Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6d036656b07e097558b155568154045e5fb5b11f","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/castorini/Anserini/pulls/185/commits","review_comments_url":"https://api.github.com/repos/castorini/Anserini/pulls/185/comments","review_comment_url":"https://api.github.com/repos/castorini/Anserini/pulls/comments{/number}","comments_url":"https://api.github.com/repos/castorini/Anserini/issues/185/comments","statuses_url":"https://api.github.com/repos/castorini/Anserini/statuses/efb434386a5ffdd85c22e614060edf3d245b3c77","head":{"label":"rosequ:wmd-scorer","ref":"wmd-scorer","sha":"efb434386a5ffdd85c22e614060edf3d245b3c77","user":{"login":"rosequ","id":8274617,"avatar_url":"https://avatars0.githubusercontent.com/u/8274617?v=3","gravatar_id":"","url":"https://api.github.com/users/rosequ","html_url":"https://github.com/rosequ","followers_url":"https://api.github.com/users/rosequ/followers","following_url":"https://api.github.com/users/rosequ/following{/other_user}","gists_url":"https://api.github.com/users/rosequ/gists{/gist_id}","starred_url":"https://api.github.com/users/rosequ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rosequ/subscriptions","organizations_url":"https://api.github.com/users/rosequ/orgs","repos_url":"https://api.github.com/users/rosequ/repos","events_url":"https://api.github.com/users/rosequ/events{/privacy}","received_events_url":"https://api.github.com/users/rosequ/received_events","type":"User","site_admin":false},"repo":{"id":75952583,"name":"Anserini","full_name":"rosequ/Anserini","owner":{"login":"rosequ","id":8274617,"avatar_url":"https://avatars0.githubusercontent.com/u/8274617?v=3","gravatar_id":"","url":"https://api.github.com/users/rosequ","html_url":"https://github.com/rosequ","followers_url":"https://api.github.com/users/rosequ/followers","following_url":"https://api.github.com/users/rosequ/following{/other_user}","gists_url":"https://api.github.com/users/rosequ/gists{/gist_id}","starred_url":"https://api.github.com/users/rosequ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rosequ/subscriptions","organizations_url":"https://api.github.com/users/rosequ/orgs","repos_url":"https://api.github.com/users/rosequ/repos","events_url":"https://api.github.com/users/rosequ/events{/privacy}","received_events_url":"https://api.github.com/users/rosequ/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rosequ/Anserini","description":"An information retrieval toolkit built on Lucene","fork":true,"url":"https://api.github.com/repos/rosequ/Anserini","forks_url":"https://api.github.com/repos/rosequ/Anserini/forks","keys_url":"https://api.github.com/repos/rosequ/Anserini/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rosequ/Anserini/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rosequ/Anserini/teams","hooks_url":"https://api.github.com/repos/rosequ/Anserini/hooks","issue_events_url":"https://api.github.com/repos/rosequ/Anserini/issues/events{/number}","events_url":"https://api.github.com/repos/rosequ/Anserini/events","assignees_url":"https://api.github.com/repos/rosequ/Anserini/assignees{/user}","branches_url":"https://api.github.com/repos/rosequ/Anserini/branches{/branch}","tags_url":"https://api.github.com/repos/rosequ/Anserini/tags","blobs_url":"https://api.github.com/repos/rosequ/Anserini/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rosequ/Anserini/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rosequ/Anserini/git/refs{/sha}","trees_url":"https://api.github.com/repos/rosequ/Anserini/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rosequ/Anserini/statuses/{sha}","languages_url":"https://api.github.com/repos/rosequ/Anserini/languages","stargazers_url":"https://api.github.com/repos/rosequ/Anserini/stargazers","contributors_url":"https://api.github.com/repos/rosequ/Anserini/contributors","subscribers_url":"https://api.github.com/repos/rosequ/Anserini/subscribers","subscription_url":"https://api.github.com/repos/rosequ/Anserini/subscription","commits_url":"https://api.github.com/repos/rosequ/Anserini/commits{/sha}","git_commits_url":"https://api.github.com/repos/rosequ/Anserini/git/commits{/sha}","comments_url":"https://api.github.com/repos/rosequ/Anserini/comments{/number}","issue_comment_url":"https://api.github.com/repos/rosequ/Anserini/issues/comments{/number}","contents_url":"https://api.github.com/repos/rosequ/Anserini/contents/{+path}","compare_url":"https://api.github.com/repos/rosequ/Anserini/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rosequ/Anserini/merges","archive_url":"https://api.github.com/repos/rosequ/Anserini/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rosequ/Anserini/downloads","issues_url":"https://api.github.com/repos/rosequ/Anserini/issues{/number}","pulls_url":"https://api.github.com/repos/rosequ/Anserini/pulls{/number}","milestones_url":"https://api.github.com/repos/rosequ/Anserini/milestones{/number}","notifications_url":"https://api.github.com/repos/rosequ/Anserini/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rosequ/Anserini/labels{/name}","releases_url":"https://api.github.com/repos/rosequ/Anserini/releases{/id}","deployments_url":"https://api.github.com/repos/rosequ/Anserini/deployments","created_at":"2016-12-08T15:56:16Z","updated_at":"2016-12-13T21:49:48Z","pushed_at":"2017-05-13T03:27:20Z","git_url":"git://github.com/rosequ/Anserini.git","ssh_url":"git@github.com:rosequ/Anserini.git","clone_url":"https://github.com/rosequ/Anserini.git","svn_url":"https://github.com/rosequ/Anserini","homepage":"","size":8545,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"castorini:master","ref":"master","sha":"c988373804707ea6bb4754e31cc0fd0cafeefa9b","user":{"login":"castorini","id":26842848,"avatar_url":"https://avatars2.githubusercontent.com/u/26842848?v=3","gravatar_id":"","url":"https://api.github.com/users/castorini","html_url":"https://github.com/castorini","followers_url":"https://api.github.com/users/castorini/followers","following_url":"https://api.github.com/users/castorini/following{/other_user}","gists_url":"https://api.github.com/users/castorini/gists{/gist_id}","starred_url":"https://api.github.com/users/castorini/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/castorini/subscriptions","organizations_url":"https://api.github.com/users/castorini/orgs","repos_url":"https://api.github.com/users/castorini/repos","events_url":"https://api.github.com/users/castorini/events{/privacy}","received_events_url":"https://api.github.com/users/castorini/received_events","type":"Organization","site_admin":false},"repo":{"id":43719677,"name":"Anserini","full_name":"castorini/Anserini","owner":{"login":"castorini","id":26842848,"avatar_url":"https://avatars2.githubusercontent.com/u/26842848?v=3","gravatar_id":"","url":"https://api.github.com/users/castorini","html_url":"https://github.com/castorini","followers_url":"https://api.github.com/users/castorini/followers","following_url":"https://api.github.com/users/castorini/following{/other_user}","gists_url":"https://api.github.com/users/castorini/gists{/gist_id}","starred_url":"https://api.github.com/users/castorini/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/castorini/subscriptions","organizations_url":"https://api.github.com/users/castorini/orgs","repos_url":"https://api.github.com/users/castorini/repos","events_url":"https://api.github.com/users/castorini/events{/privacy}","received_events_url":"https://api.github.com/users/castorini/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/castorini/Anserini","description":"An information retrieval toolkit built on Lucene","fork":false,"url":"https://api.github.com/repos/castorini/Anserini","forks_url":"https://api.github.com/repos/castorini/Anserini/forks","keys_url":"https://api.github.com/repos/castorini/Anserini/keys{/key_id}","collaborators_url":"https://api.github.com/repos/castorini/Anserini/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/castorini/Anserini/teams","hooks_url":"https://api.github.com/repos/castorini/Anserini/hooks","issue_events_url":"https://api.github.com/repos/castorini/Anserini/issues/events{/number}","events_url":"https://api.github.com/repos/castorini/Anserini/events","assignees_url":"https://api.github.com/repos/castorini/Anserini/assignees{/user}","branches_url":"https://api.github.com/repos/castorini/Anserini/branches{/branch}","tags_url":"https://api.github.com/repos/castorini/Anserini/tags","blobs_url":"https://api.github.com/repos/castorini/Anserini/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/castorini/Anserini/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/castorini/Anserini/git/refs{/sha}","trees_url":"https://api.github.com/repos/castorini/Anserini/git/trees{/sha}","statuses_url":"https://api.github.com/repos/castorini/Anserini/statuses/{sha}","languages_url":"https://api.github.com/repos/castorini/Anserini/languages","stargazers_url":"https://api.github.com/repos/castorini/Anserini/stargazers","contributors_url":"https://api.github.com/repos/castorini/Anserini/contributors","subscribers_url":"https://api.github.com/repos/castorini/Anserini/subscribers","subscription_url":"https://api.github.com/repos/castorini/Anserini/subscription","commits_url":"https://api.github.com/repos/castorini/Anserini/commits{/sha}","git_commits_url":"https://api.github.com/repos/castorini/Anserini/git/commits{/sha}","comments_url":"https://api.github.com/repos/castorini/Anserini/comments{/number}","issue_comment_url":"https://api.github.com/repos/castorini/Anserini/issues/comments{/number}","contents_url":"https://api.github.com/repos/castorini/Anserini/contents/{+path}","compare_url":"https://api.github.com/repos/castorini/Anserini/compare/{base}...{head}","merges_url":"https://api.github.com/repos/castorini/Anserini/merges","archive_url":"https://api.github.com/repos/castorini/Anserini/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/castorini/Anserini/downloads","issues_url":"https://api.github.com/repos/castorini/Anserini/issues{/number}","pulls_url":"https://api.github.com/repos/castorini/Anserini/pulls{/number}","milestones_url":"https://api.github.com/repos/castorini/Anserini/milestones{/number}","notifications_url":"https://api.github.com/repos/castorini/Anserini/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/castorini/Anserini/labels{/name}","releases_url":"https://api.github.com/repos/castorini/Anserini/releases{/id}","deployments_url":"https://api.github.com/repos/castorini/Anserini/deployments","created_at":"2015-10-05T23:27:54Z","updated_at":"2017-04-13T17:49:13Z","pushed_at":"2017-05-13T03:38:00Z","git_url":"git://github.com/castorini/Anserini.git","ssh_url":"git@github.com:castorini/Anserini.git","clone_url":"https://github.com/castorini/Anserini.git","svn_url":"https://github.com/castorini/Anserini","homepage":"http://anserini.io/","size":8466,"stargazers_count":22,"watchers_count":22,"language":"Java","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":31,"mirror_url":null,"open_issues_count":24,"forks":31,"open_issues":24,"watchers":22,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/castorini/Anserini/pulls/185"},"html":{"href":"https://github.com/castorini/Anserini/pull/185"},"issue":{"href":"https://api.github.com/repos/castorini/Anserini/issues/185"},"comments":{"href":"https://api.github.com/repos/castorini/Anserini/issues/185/comments"},"review_comments":{"href":"https://api.github.com/repos/castorini/Anserini/pulls/185/comments"},"review_comment":{"href":"https://api.github.com/repos/castorini/Anserini/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/castorini/Anserini/pulls/185/commits"},"statuses":{"href":"https://api.github.com/repos/castorini/Anserini/statuses/efb434386a5ffdd85c22e614060edf3d245b3c77"}}}}
{"actor":{"display_login":"minafarid"}}