{"action":"created","comment":{"url":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/comments/139275380","pull_request_review_id":63191738,"id":139275380,"diff_hunk":"@@ -0,0 +1,329 @@\n+#!/usr/bin/env python\n+# encoding: utf-8\n+import warnings\n+import numpy as np\n+\n+from collections import OrderedDict\n+from gensim import utils\n+from six import string_types\n+\n+\n+\"\"\"\n+Produce translation matrix to translate the word from one language to another language, using either\n+standard nearest neighbour method or globally corrected neighbour retrieval method [1].\n+\n+This method can be used to augment the existing phrase tables with more candidate translations, or\n+filter out errors from the translation tables and known dictionaries [2].\n+\n+Initialize a model with e.g.::\n+\n+    >>> transmat = TranslationMatrix(word_pair, source_word_vec, target_word_vec)\n+\n+Train a model with e.g.::\n+\n+    >>> transmat.train(word_pair)\n+\n+Persist a model to disk with::\n+\n+    >>> transmat.save(fname)\n+    >>> transmat  = TranslationMatrix.load(fname)\n+\n+Translate the source words to target words, for example\n+\n+    >>> transmat.translate([\"one\", \"two\", \"three\"], topn=3)\n+\n+.. [1] Dinu, Georgiana, Angeliki Lazaridou, and Marco Baroni. \"Improving zero-shot learning by mitigating the\n+        hubness problem.\" arXiv preprint arXiv:1412.6568 (2014).\n+.. [2] Tomas Mikolov, Ilya Sutskever, Kai Chen, Greg Corrado, and Jeffrey Dean. Distributed Representations of Words and Phrases and their Compositionality.\n+       In Proceedings of NIPS, 2013.\n+\"\"\"\n+\n+\n+class Space(object):\n+    \"\"\"\n+    An auxiliary class for storing the the words space\n+\n+    Attributes:\n+        `mat` (ndarray): each row is the word vector of the lexicon\n+        `index2word` (list): a list of lexicon\n+        `word2index` (dict): map the word to index\n+    \"\"\"\n+    def __init__(self, matrix, index2word):\n+        \"\"\"\n+        `matrix`: N * length_of_word_vec, which store the word's vector\n+        `index2word`: a list of words in the `Space` object\n+        `word2index`: a dict which for word indexing\n+        \"\"\"\n+        self.mat = matrix\n+        self.index2word = index2word\n+\n+        # build a dict to map word to index\n+        self.word2index = {}\n+        for idx, word in enumerate(self.index2word):\n+            self.word2index[word] = idx\n+\n+    @classmethod\n+    def build(cls, lang_vec, lexicon=None):\n+        \"\"\"\n+        construct a space class for the lexicon, if it's provided.\n+        Args:\n+            `lang_vec`: word2vec model that extract word vector for lexicon\n+            `lexicon`: the default is None, if it is not provided, the lexicon\n+                    is all the lang_vec's word, i.e. lang_vec.vocab.keys()\n+        Returns:\n+            `Space` object for the lexicon\n+        \"\"\"\n+        # `words` to store all the word that\n+        # `mat` to store all the word vector for the word in 'words' list\n+        words = []\n+        mat = []\n+        if lexicon is not None:\n+            # if the lexicon is not provided, using the all the Keyedvectors's words as default\n+            for item in lexicon:\n+                words.append(item)\n+                mat.append(lang_vec.syn0[lang_vec.vocab[item].index])\n+\n+        else:\n+            for item in lang_vec.vocab.keys():\n+                words.append(item)\n+                mat.append(lang_vec.syn0[lang_vec.vocab[item].index])\n+\n+        return Space(mat, words)\n+\n+    def normalize(self):\n+        \"\"\" normalized the word vector's matrix \"\"\"\n+        self.mat = self.mat / np.sqrt(np.sum(np.multiply(self.mat, self.mat), axis=1, keepdims=True))\n+\n+\n+class TranslationMatrix(utils.SaveLoad):\n+    \"\"\"\n+    Objects of this class realize the translation matrix which map the source language\n+    to the target language.\n+     The main methods are:\n+\n+    1. constructor,\n+    2. the `train` method, which initialize everything needed to build a translation matrix\n+    3. the `translate` method, which given new word and its vector representation,\n+    we map it to the other language space by computing z = Wx, then return the\n+    word whose representation is close to z.\n+\n+    the details use seen the notebook (translation_matrix.ipynb)\n+\n+    >>> transmat = TranslationMatrix(word_pair, source_lang_vec, target_lang_vec)\n+    >>> transmat.train(word_pair)\n+    >>> translated_word = transmat.translate(words, topn=3)\n+\n+    \"\"\"\n+    def __init__(self, word_pair, source_lang_vec, target_lang_vec, random_state=None):\n+        \"\"\"\n+        Initialize the model from a list pair of `word_pair`. Each word_pair is tupe\n+         with source language word and target language word.\n+\n+        Examples: [(\"one\", \"uno\"), (\"two\", \"due\")]\n+\n+        Args:\n+            `word_pair` (list): a list pair of words\n+            `source_lang_vec` (KeyedVectors): a set of word vector of source language\n+            `target_lang_vec` (KeyedVectors): a set of word vector of target language\n+        \"\"\"\n+\n+        if len(word_pair[0]) != 2:\n+            raise ValueError(\"Each training data item must contain two different language words.\")\n+\n+        self.source_word, self.target_word = zip(*word_pair)\n+        self.source_lang_vec = source_lang_vec\n+        self.target_lang_vec = target_lang_vec\n+\n+        self.random_state = utils.get_random_state(random_state)\n+        self.translation_matrix = None\n+        self.source_space = None\n+        self.target_space = None\n+\n+    def train(self, word_pair):","path":"gensim/models/translation_matrix.py","position":142,"original_position":142,"commit_id":"41e5584fe52b6abea2ea22fb600dbe0431344441","original_commit_id":"41e5584fe52b6abea2ea22fb600dbe0431344441","user":{"login":"robotcator","id":5803439,"avatar_url":"https://avatars2.githubusercontent.com/u/5803439?v=4","gravatar_id":"","url":"https://api.github.com/users/robotcator","html_url":"https://github.com/robotcator","followers_url":"https://api.github.com/users/robotcator/followers","following_url":"https://api.github.com/users/robotcator/following{/other_user}","gists_url":"https://api.github.com/users/robotcator/gists{/gist_id}","starred_url":"https://api.github.com/users/robotcator/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robotcator/subscriptions","organizations_url":"https://api.github.com/users/robotcator/orgs","repos_url":"https://api.github.com/users/robotcator/repos","events_url":"https://api.github.com/users/robotcator/events{/privacy}","received_events_url":"https://api.github.com/users/robotcator/received_events","type":"User","site_admin":false},"body":"As you said in the second comment, if training was combined with initialization, then this parameter would be used in `train` method.","created_at":"2017-09-16T01:54:15Z","updated_at":"2017-09-16T01:54:15Z","html_url":"https://github.com/RaRe-Technologies/gensim/pull/1434#discussion_r139275380","pull_request_url":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/1434","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/comments/139275380"},"html":{"href":"https://github.com/RaRe-Technologies/gensim/pull/1434#discussion_r139275380"},"pull_request":{"href":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/1434"}},"in_reply_to_id":138752017},"pull_request":{"url":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/1434","id":126499273,"html_url":"https://github.com/RaRe-Technologies/gensim/pull/1434","diff_url":"https://github.com/RaRe-Technologies/gensim/pull/1434.diff","patch_url":"https://github.com/RaRe-Technologies/gensim/pull/1434.patch","issue_url":"https://api.github.com/repos/RaRe-Technologies/gensim/issues/1434","number":1434,"state":"closed","locked":false,"title":"[MRG] Implement 'Translation Matrix' ","user":{"login":"robotcator","id":5803439,"avatar_url":"https://avatars2.githubusercontent.com/u/5803439?v=4","gravatar_id":"","url":"https://api.github.com/users/robotcator","html_url":"https://github.com/robotcator","followers_url":"https://api.github.com/users/robotcator/followers","following_url":"https://api.github.com/users/robotcator/following{/other_user}","gists_url":"https://api.github.com/users/robotcator/gists{/gist_id}","starred_url":"https://api.github.com/users/robotcator/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robotcator/subscriptions","organizations_url":"https://api.github.com/users/robotcator/orgs","repos_url":"https://api.github.com/users/robotcator/repos","events_url":"https://api.github.com/users/robotcator/events{/privacy}","received_events_url":"https://api.github.com/users/robotcator/received_events","type":"User","site_admin":false},"body":"This PR adds translation matrix feature to gensim model.","created_at":"2017-06-20T10:33:27Z","updated_at":"2017-09-16T01:54:15Z","closed_at":"2017-09-13T12:38:25Z","merged_at":"2017-09-13T12:38:25Z","merge_commit_sha":"1c0098c83a9ba3e79b88a71a8c18c7f3ea6e26d4","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/1434/commits","review_comments_url":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/1434/comments","review_comment_url":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/comments{/number}","comments_url":"https://api.github.com/repos/RaRe-Technologies/gensim/issues/1434/comments","statuses_url":"https://api.github.com/repos/RaRe-Technologies/gensim/statuses/41e5584fe52b6abea2ea22fb600dbe0431344441","head":{"label":"robotcator:transmat","ref":"transmat","sha":"41e5584fe52b6abea2ea22fb600dbe0431344441","user":{"login":"robotcator","id":5803439,"avatar_url":"https://avatars2.githubusercontent.com/u/5803439?v=4","gravatar_id":"","url":"https://api.github.com/users/robotcator","html_url":"https://github.com/robotcator","followers_url":"https://api.github.com/users/robotcator/followers","following_url":"https://api.github.com/users/robotcator/following{/other_user}","gists_url":"https://api.github.com/users/robotcator/gists{/gist_id}","starred_url":"https://api.github.com/users/robotcator/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robotcator/subscriptions","organizations_url":"https://api.github.com/users/robotcator/orgs","repos_url":"https://api.github.com/users/robotcator/repos","events_url":"https://api.github.com/users/robotcator/events{/privacy}","received_events_url":"https://api.github.com/users/robotcator/received_events","type":"User","site_admin":false},"repo":{"id":83872882,"name":"gensim","full_name":"robotcator/gensim","owner":{"login":"robotcator","id":5803439,"avatar_url":"https://avatars2.githubusercontent.com/u/5803439?v=4","gravatar_id":"","url":"https://api.github.com/users/robotcator","html_url":"https://github.com/robotcator","followers_url":"https://api.github.com/users/robotcator/followers","following_url":"https://api.github.com/users/robotcator/following{/other_user}","gists_url":"https://api.github.com/users/robotcator/gists{/gist_id}","starred_url":"https://api.github.com/users/robotcator/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robotcator/subscriptions","organizations_url":"https://api.github.com/users/robotcator/orgs","repos_url":"https://api.github.com/users/robotcator/repos","events_url":"https://api.github.com/users/robotcator/events{/privacy}","received_events_url":"https://api.github.com/users/robotcator/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/robotcator/gensim","description":"Topic Modelling for Humans","fork":true,"url":"https://api.github.com/repos/robotcator/gensim","forks_url":"https://api.github.com/repos/robotcator/gensim/forks","keys_url":"https://api.github.com/repos/robotcator/gensim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/robotcator/gensim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/robotcator/gensim/teams","hooks_url":"https://api.github.com/repos/robotcator/gensim/hooks","issue_events_url":"https://api.github.com/repos/robotcator/gensim/issues/events{/number}","events_url":"https://api.github.com/repos/robotcator/gensim/events","assignees_url":"https://api.github.com/repos/robotcator/gensim/assignees{/user}","branches_url":"https://api.github.com/repos/robotcator/gensim/branches{/branch}","tags_url":"https://api.github.com/repos/robotcator/gensim/tags","blobs_url":"https://api.github.com/repos/robotcator/gensim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/robotcator/gensim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/robotcator/gensim/git/refs{/sha}","trees_url":"https://api.github.com/repos/robotcator/gensim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/robotcator/gensim/statuses/{sha}","languages_url":"https://api.github.com/repos/robotcator/gensim/languages","stargazers_url":"https://api.github.com/repos/robotcator/gensim/stargazers","contributors_url":"https://api.github.com/repos/robotcator/gensim/contributors","subscribers_url":"https://api.github.com/repos/robotcator/gensim/subscribers","subscription_url":"https://api.github.com/repos/robotcator/gensim/subscription","commits_url":"https://api.github.com/repos/robotcator/gensim/commits{/sha}","git_commits_url":"https://api.github.com/repos/robotcator/gensim/git/commits{/sha}","comments_url":"https://api.github.com/repos/robotcator/gensim/comments{/number}","issue_comment_url":"https://api.github.com/repos/robotcator/gensim/issues/comments{/number}","contents_url":"https://api.github.com/repos/robotcator/gensim/contents/{+path}","compare_url":"https://api.github.com/repos/robotcator/gensim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/robotcator/gensim/merges","archive_url":"https://api.github.com/repos/robotcator/gensim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/robotcator/gensim/downloads","issues_url":"https://api.github.com/repos/robotcator/gensim/issues{/number}","pulls_url":"https://api.github.com/repos/robotcator/gensim/pulls{/number}","milestones_url":"https://api.github.com/repos/robotcator/gensim/milestones{/number}","notifications_url":"https://api.github.com/repos/robotcator/gensim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/robotcator/gensim/labels{/name}","releases_url":"https://api.github.com/repos/robotcator/gensim/releases{/id}","deployments_url":"https://api.github.com/repos/robotcator/gensim/deployments","created_at":"2017-03-04T06:53:06Z","updated_at":"2017-03-04T06:53:09Z","pushed_at":"2017-09-11T09:24:40Z","git_url":"git://github.com/robotcator/gensim.git","ssh_url":"git@github.com:robotcator/gensim.git","clone_url":"https://github.com/robotcator/gensim.git","svn_url":"https://github.com/robotcator/gensim","homepage":"http://radimrehurek.com/gensim/","size":43283,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"develop"}},"base":{"label":"RaRe-Technologies:develop","ref":"develop","sha":"718b1c6bd1a8a98625993d73b83d98baf385752d","user":{"login":"RaRe-Technologies","id":12515886,"avatar_url":"https://avatars3.githubusercontent.com/u/12515886?v=4","gravatar_id":"","url":"https://api.github.com/users/RaRe-Technologies","html_url":"https://github.com/RaRe-Technologies","followers_url":"https://api.github.com/users/RaRe-Technologies/followers","following_url":"https://api.github.com/users/RaRe-Technologies/following{/other_user}","gists_url":"https://api.github.com/users/RaRe-Technologies/gists{/gist_id}","starred_url":"https://api.github.com/users/RaRe-Technologies/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/RaRe-Technologies/subscriptions","organizations_url":"https://api.github.com/users/RaRe-Technologies/orgs","repos_url":"https://api.github.com/users/RaRe-Technologies/repos","events_url":"https://api.github.com/users/RaRe-Technologies/events{/privacy}","received_events_url":"https://api.github.com/users/RaRe-Technologies/received_events","type":"Organization","site_admin":false},"repo":{"id":1349775,"name":"gensim","full_name":"RaRe-Technologies/gensim","owner":{"login":"RaRe-Technologies","id":12515886,"avatar_url":"https://avatars3.githubusercontent.com/u/12515886?v=4","gravatar_id":"","url":"https://api.github.com/users/RaRe-Technologies","html_url":"https://github.com/RaRe-Technologies","followers_url":"https://api.github.com/users/RaRe-Technologies/followers","following_url":"https://api.github.com/users/RaRe-Technologies/following{/other_user}","gists_url":"https://api.github.com/users/RaRe-Technologies/gists{/gist_id}","starred_url":"https://api.github.com/users/RaRe-Technologies/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/RaRe-Technologies/subscriptions","organizations_url":"https://api.github.com/users/RaRe-Technologies/orgs","repos_url":"https://api.github.com/users/RaRe-Technologies/repos","events_url":"https://api.github.com/users/RaRe-Technologies/events{/privacy}","received_events_url":"https://api.github.com/users/RaRe-Technologies/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/RaRe-Technologies/gensim","description":"Topic Modelling for Humans","fork":false,"url":"https://api.github.com/repos/RaRe-Technologies/gensim","forks_url":"https://api.github.com/repos/RaRe-Technologies/gensim/forks","keys_url":"https://api.github.com/repos/RaRe-Technologies/gensim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/RaRe-Technologies/gensim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/RaRe-Technologies/gensim/teams","hooks_url":"https://api.github.com/repos/RaRe-Technologies/gensim/hooks","issue_events_url":"https://api.github.com/repos/RaRe-Technologies/gensim/issues/events{/number}","events_url":"https://api.github.com/repos/RaRe-Technologies/gensim/events","assignees_url":"https://api.github.com/repos/RaRe-Technologies/gensim/assignees{/user}","branches_url":"https://api.github.com/repos/RaRe-Technologies/gensim/branches{/branch}","tags_url":"https://api.github.com/repos/RaRe-Technologies/gensim/tags","blobs_url":"https://api.github.com/repos/RaRe-Technologies/gensim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/RaRe-Technologies/gensim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/RaRe-Technologies/gensim/git/refs{/sha}","trees_url":"https://api.github.com/repos/RaRe-Technologies/gensim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/RaRe-Technologies/gensim/statuses/{sha}","languages_url":"https://api.github.com/repos/RaRe-Technologies/gensim/languages","stargazers_url":"https://api.github.com/repos/RaRe-Technologies/gensim/stargazers","contributors_url":"https://api.github.com/repos/RaRe-Technologies/gensim/contributors","subscribers_url":"https://api.github.com/repos/RaRe-Technologies/gensim/subscribers","subscription_url":"https://api.github.com/repos/RaRe-Technologies/gensim/subscription","commits_url":"https://api.github.com/repos/RaRe-Technologies/gensim/commits{/sha}","git_commits_url":"https://api.github.com/repos/RaRe-Technologies/gensim/git/commits{/sha}","comments_url":"https://api.github.com/repos/RaRe-Technologies/gensim/comments{/number}","issue_comment_url":"https://api.github.com/repos/RaRe-Technologies/gensim/issues/comments{/number}","contents_url":"https://api.github.com/repos/RaRe-Technologies/gensim/contents/{+path}","compare_url":"https://api.github.com/repos/RaRe-Technologies/gensim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/RaRe-Technologies/gensim/merges","archive_url":"https://api.github.com/repos/RaRe-Technologies/gensim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/RaRe-Technologies/gensim/downloads","issues_url":"https://api.github.com/repos/RaRe-Technologies/gensim/issues{/number}","pulls_url":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls{/number}","milestones_url":"https://api.github.com/repos/RaRe-Technologies/gensim/milestones{/number}","notifications_url":"https://api.github.com/repos/RaRe-Technologies/gensim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/RaRe-Technologies/gensim/labels{/name}","releases_url":"https://api.github.com/repos/RaRe-Technologies/gensim/releases{/id}","deployments_url":"https://api.github.com/repos/RaRe-Technologies/gensim/deployments","created_at":"2011-02-10T07:43:04Z","updated_at":"2017-09-15T21:48:02Z","pushed_at":"2017-09-14T11:01:38Z","git_url":"git://github.com/RaRe-Technologies/gensim.git","ssh_url":"git@github.com:RaRe-Technologies/gensim.git","clone_url":"https://github.com/RaRe-Technologies/gensim.git","svn_url":"https://github.com/RaRe-Technologies/gensim","homepage":"http://radimrehurek.com/gensim/","size":46577,"stargazers_count":5039,"watchers_count":5039,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2006,"mirror_url":null,"open_issues_count":211,"forks":2006,"open_issues":211,"watchers":5039,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/1434"},"html":{"href":"https://github.com/RaRe-Technologies/gensim/pull/1434"},"issue":{"href":"https://api.github.com/repos/RaRe-Technologies/gensim/issues/1434"},"comments":{"href":"https://api.github.com/repos/RaRe-Technologies/gensim/issues/1434/comments"},"review_comments":{"href":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/1434/comments"},"review_comment":{"href":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/RaRe-Technologies/gensim/pulls/1434/commits"},"statuses":{"href":"https://api.github.com/repos/RaRe-Technologies/gensim/statuses/41e5584fe52b6abea2ea22fb600dbe0431344441"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"robotcator"}}