{"action":"created","comment":{"url":"https://api.github.com/repos/medicode/tensor2tensor/pulls/comments/150655755","pull_request_review_id":76238428,"id":150655755,"diff_hunk":"@@ -109,7 +110,83 @@\n     \"Comma-separated list of name=value pairs to control decode behavior. \"\n     \"See decoding.decode_hparams for defaults.\")\n \n-\n+# Early stopping flags\n+flags.DEFINE_string(\"early_stopping_metric\", \"loss\",\n+                    \"Metric to use to decide on early stopping\")\n+flags.DEFINE_integer(\"early_stopping_patience\", 4,\n+                     \"How many iterations we will wait before doing early stopping.\")\n+flags.DEFINE_integer(\"train_steps_per_iteration\", 100,\n+                     \"How many training steps count as an iterations when doing early\"\n+                     \"stopping.\")\n+\n+\n+class EarlyStoppingExperiment(tf.contrib.learn.Experiment):\n+  def __init__(self, *args, **kwargs):\n+    super(EarlyStoppingExperiment, self).__init__(\n+      *args,\n+      train_steps_per_iteration=FLAGS.train_steps_per_iteration,\n+      **kwargs)\n+    self.patience = FLAGS.early_stopping_patience\n+    self.metric = FLAGS.early_stopping_metric\n+\n+    # For loss, we want low numbers, for all others we want high\n+    # numbers\n+    if self.metric == 'loss':\n+      self.high_optimal = False\n+    else:\n+      self.high_optimal = True\n+\n+    # The most recent values seen; initialize with the worst possible\n+    # values so that we don't stop until we have filled up the\n+    # values_seen buffer with real values.\n+    if self.high_optimal:\n+      self.values_seen = np.zeros(self.patience) - np.inf\n+    else:\n+      self.values_seen = np.zeros(self.patience) + np.inf\n+      \n+  def _early_stopping_predicate(self, results):\n+    \"\"\"Predicate that returns True if we should continue based on the\n+    results seen so far.\n+\n+    Args:\n+      results: dict mapping metric names to values\n+\n+    Returns:\n+      return_value: True if we should keep training, False if we\n+                    should stop\n+    \"\"\"\n+    # The first time this gets called, results will be None, and we\n+    # don't want to stop\n+    if not results:\n+      return True\n+\n+    # Add new value and forget oldest value\n+    self.values_seen[:-1] = self.values_seen[1:]\n+    self.values_seen[-1] = results[self.metric]\n+\n+    print(self.values_seen)","path":"tensor2tensor/utils/trainer_utils.py","position":67,"original_position":67,"commit_id":"2f99e2090bf62582d4ba914c919412a9a133dccd","original_commit_id":"2f99e2090bf62582d4ba914c919412a9a133dccd","user":{"login":"epurdyf","id":32338500,"avatar_url":"https://avatars1.githubusercontent.com/u/32338500?v=4","gravatar_id":"","url":"https://api.github.com/users/epurdyf","html_url":"https://github.com/epurdyf","followers_url":"https://api.github.com/users/epurdyf/followers","following_url":"https://api.github.com/users/epurdyf/following{/other_user}","gists_url":"https://api.github.com/users/epurdyf/gists{/gist_id}","starred_url":"https://api.github.com/users/epurdyf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/epurdyf/subscriptions","organizations_url":"https://api.github.com/users/epurdyf/orgs","repos_url":"https://api.github.com/users/epurdyf/repos","events_url":"https://api.github.com/users/epurdyf/events{/privacy}","received_events_url":"https://api.github.com/users/epurdyf/received_events","type":"User","site_admin":false},"body":"Removed.","created_at":"2017-11-13T20:29:09Z","updated_at":"2017-11-13T20:29:09Z","html_url":"https://github.com/medicode/tensor2tensor/pull/7#discussion_r150655755","pull_request_url":"https://api.github.com/repos/medicode/tensor2tensor/pulls/7","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/medicode/tensor2tensor/pulls/comments/150655755"},"html":{"href":"https://github.com/medicode/tensor2tensor/pull/7#discussion_r150655755"},"pull_request":{"href":"https://api.github.com/repos/medicode/tensor2tensor/pulls/7"}},"in_reply_to_id":150368630},"pull_request":{"url":"https://api.github.com/repos/medicode/tensor2tensor/pulls/7","id":152029433,"html_url":"https://github.com/medicode/tensor2tensor/pull/7","diff_url":"https://github.com/medicode/tensor2tensor/pull/7.diff","patch_url":"https://github.com/medicode/tensor2tensor/pull/7.patch","issue_url":"https://api.github.com/repos/medicode/tensor2tensor/issues/7","number":7,"state":"open","locked":false,"title":"Early stopping etc.","user":{"login":"epurdyf","id":32338500,"avatar_url":"https://avatars1.githubusercontent.com/u/32338500?v=4","gravatar_id":"","url":"https://api.github.com/users/epurdyf","html_url":"https://github.com/epurdyf","followers_url":"https://api.github.com/users/epurdyf/followers","following_url":"https://api.github.com/users/epurdyf/following{/other_user}","gists_url":"https://api.github.com/users/epurdyf/gists{/gist_id}","starred_url":"https://api.github.com/users/epurdyf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/epurdyf/subscriptions","organizations_url":"https://api.github.com/users/epurdyf/orgs","repos_url":"https://api.github.com/users/epurdyf/repos","events_url":"https://api.github.com/users/epurdyf/events{/privacy}","received_events_url":"https://api.github.com/users/epurdyf/received_events","type":"User","site_admin":false},"body":"Early stopping, profiling, bug fix, emitting multiple outputs","created_at":"2017-11-11T00:21:47Z","updated_at":"2017-11-13T20:29:09Z","closed_at":null,"merged_at":null,"merge_commit_sha":null,"assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/medicode/tensor2tensor/pulls/7/commits","review_comments_url":"https://api.github.com/repos/medicode/tensor2tensor/pulls/7/comments","review_comment_url":"https://api.github.com/repos/medicode/tensor2tensor/pulls/comments{/number}","comments_url":"https://api.github.com/repos/medicode/tensor2tensor/issues/7/comments","statuses_url":"https://api.github.com/repos/medicode/tensor2tensor/statuses/2f99e2090bf62582d4ba914c919412a9a133dccd","head":{"label":"medicode:feature/early_stopping","ref":"feature/early_stopping","sha":"2f99e2090bf62582d4ba914c919412a9a133dccd","user":{"login":"medicode","id":22822202,"avatar_url":"https://avatars0.githubusercontent.com/u/22822202?v=4","gravatar_id":"","url":"https://api.github.com/users/medicode","html_url":"https://github.com/medicode","followers_url":"https://api.github.com/users/medicode/followers","following_url":"https://api.github.com/users/medicode/following{/other_user}","gists_url":"https://api.github.com/users/medicode/gists{/gist_id}","starred_url":"https://api.github.com/users/medicode/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medicode/subscriptions","organizations_url":"https://api.github.com/users/medicode/orgs","repos_url":"https://api.github.com/users/medicode/repos","events_url":"https://api.github.com/users/medicode/events{/privacy}","received_events_url":"https://api.github.com/users/medicode/received_events","type":"Organization","site_admin":false},"repo":{"id":105474322,"name":"tensor2tensor","full_name":"medicode/tensor2tensor","owner":{"login":"medicode","id":22822202,"avatar_url":"https://avatars0.githubusercontent.com/u/22822202?v=4","gravatar_id":"","url":"https://api.github.com/users/medicode","html_url":"https://github.com/medicode","followers_url":"https://api.github.com/users/medicode/followers","following_url":"https://api.github.com/users/medicode/following{/other_user}","gists_url":"https://api.github.com/users/medicode/gists{/gist_id}","starred_url":"https://api.github.com/users/medicode/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medicode/subscriptions","organizations_url":"https://api.github.com/users/medicode/orgs","repos_url":"https://api.github.com/users/medicode/repos","events_url":"https://api.github.com/users/medicode/events{/privacy}","received_events_url":"https://api.github.com/users/medicode/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/medicode/tensor2tensor","description":"A library for generalized sequence to sequence models","fork":true,"url":"https://api.github.com/repos/medicode/tensor2tensor","forks_url":"https://api.github.com/repos/medicode/tensor2tensor/forks","keys_url":"https://api.github.com/repos/medicode/tensor2tensor/keys{/key_id}","collaborators_url":"https://api.github.com/repos/medicode/tensor2tensor/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/medicode/tensor2tensor/teams","hooks_url":"https://api.github.com/repos/medicode/tensor2tensor/hooks","issue_events_url":"https://api.github.com/repos/medicode/tensor2tensor/issues/events{/number}","events_url":"https://api.github.com/repos/medicode/tensor2tensor/events","assignees_url":"https://api.github.com/repos/medicode/tensor2tensor/assignees{/user}","branches_url":"https://api.github.com/repos/medicode/tensor2tensor/branches{/branch}","tags_url":"https://api.github.com/repos/medicode/tensor2tensor/tags","blobs_url":"https://api.github.com/repos/medicode/tensor2tensor/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/medicode/tensor2tensor/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/medicode/tensor2tensor/git/refs{/sha}","trees_url":"https://api.github.com/repos/medicode/tensor2tensor/git/trees{/sha}","statuses_url":"https://api.github.com/repos/medicode/tensor2tensor/statuses/{sha}","languages_url":"https://api.github.com/repos/medicode/tensor2tensor/languages","stargazers_url":"https://api.github.com/repos/medicode/tensor2tensor/stargazers","contributors_url":"https://api.github.com/repos/medicode/tensor2tensor/contributors","subscribers_url":"https://api.github.com/repos/medicode/tensor2tensor/subscribers","subscription_url":"https://api.github.com/repos/medicode/tensor2tensor/subscription","commits_url":"https://api.github.com/repos/medicode/tensor2tensor/commits{/sha}","git_commits_url":"https://api.github.com/repos/medicode/tensor2tensor/git/commits{/sha}","comments_url":"https://api.github.com/repos/medicode/tensor2tensor/comments{/number}","issue_comment_url":"https://api.github.com/repos/medicode/tensor2tensor/issues/comments{/number}","contents_url":"https://api.github.com/repos/medicode/tensor2tensor/contents/{+path}","compare_url":"https://api.github.com/repos/medicode/tensor2tensor/compare/{base}...{head}","merges_url":"https://api.github.com/repos/medicode/tensor2tensor/merges","archive_url":"https://api.github.com/repos/medicode/tensor2tensor/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/medicode/tensor2tensor/downloads","issues_url":"https://api.github.com/repos/medicode/tensor2tensor/issues{/number}","pulls_url":"https://api.github.com/repos/medicode/tensor2tensor/pulls{/number}","milestones_url":"https://api.github.com/repos/medicode/tensor2tensor/milestones{/number}","notifications_url":"https://api.github.com/repos/medicode/tensor2tensor/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/medicode/tensor2tensor/labels{/name}","releases_url":"https://api.github.com/repos/medicode/tensor2tensor/releases{/id}","deployments_url":"https://api.github.com/repos/medicode/tensor2tensor/deployments","created_at":"2017-10-01T21:04:04Z","updated_at":"2017-10-01T21:04:06Z","pushed_at":"2017-11-11T05:15:28Z","git_url":"git://github.com/medicode/tensor2tensor.git","ssh_url":"git@github.com:medicode/tensor2tensor.git","clone_url":"https://github.com/medicode/tensor2tensor.git","svn_url":"https://github.com/medicode/tensor2tensor","homepage":null,"size":1490,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"medicode:master","ref":"master","sha":"53487869b51096c55a95a0dfc4013d5814d0dbc7","user":{"login":"medicode","id":22822202,"avatar_url":"https://avatars0.githubusercontent.com/u/22822202?v=4","gravatar_id":"","url":"https://api.github.com/users/medicode","html_url":"https://github.com/medicode","followers_url":"https://api.github.com/users/medicode/followers","following_url":"https://api.github.com/users/medicode/following{/other_user}","gists_url":"https://api.github.com/users/medicode/gists{/gist_id}","starred_url":"https://api.github.com/users/medicode/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medicode/subscriptions","organizations_url":"https://api.github.com/users/medicode/orgs","repos_url":"https://api.github.com/users/medicode/repos","events_url":"https://api.github.com/users/medicode/events{/privacy}","received_events_url":"https://api.github.com/users/medicode/received_events","type":"Organization","site_admin":false},"repo":{"id":105474322,"name":"tensor2tensor","full_name":"medicode/tensor2tensor","owner":{"login":"medicode","id":22822202,"avatar_url":"https://avatars0.githubusercontent.com/u/22822202?v=4","gravatar_id":"","url":"https://api.github.com/users/medicode","html_url":"https://github.com/medicode","followers_url":"https://api.github.com/users/medicode/followers","following_url":"https://api.github.com/users/medicode/following{/other_user}","gists_url":"https://api.github.com/users/medicode/gists{/gist_id}","starred_url":"https://api.github.com/users/medicode/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medicode/subscriptions","organizations_url":"https://api.github.com/users/medicode/orgs","repos_url":"https://api.github.com/users/medicode/repos","events_url":"https://api.github.com/users/medicode/events{/privacy}","received_events_url":"https://api.github.com/users/medicode/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/medicode/tensor2tensor","description":"A library for generalized sequence to sequence models","fork":true,"url":"https://api.github.com/repos/medicode/tensor2tensor","forks_url":"https://api.github.com/repos/medicode/tensor2tensor/forks","keys_url":"https://api.github.com/repos/medicode/tensor2tensor/keys{/key_id}","collaborators_url":"https://api.github.com/repos/medicode/tensor2tensor/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/medicode/tensor2tensor/teams","hooks_url":"https://api.github.com/repos/medicode/tensor2tensor/hooks","issue_events_url":"https://api.github.com/repos/medicode/tensor2tensor/issues/events{/number}","events_url":"https://api.github.com/repos/medicode/tensor2tensor/events","assignees_url":"https://api.github.com/repos/medicode/tensor2tensor/assignees{/user}","branches_url":"https://api.github.com/repos/medicode/tensor2tensor/branches{/branch}","tags_url":"https://api.github.com/repos/medicode/tensor2tensor/tags","blobs_url":"https://api.github.com/repos/medicode/tensor2tensor/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/medicode/tensor2tensor/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/medicode/tensor2tensor/git/refs{/sha}","trees_url":"https://api.github.com/repos/medicode/tensor2tensor/git/trees{/sha}","statuses_url":"https://api.github.com/repos/medicode/tensor2tensor/statuses/{sha}","languages_url":"https://api.github.com/repos/medicode/tensor2tensor/languages","stargazers_url":"https://api.github.com/repos/medicode/tensor2tensor/stargazers","contributors_url":"https://api.github.com/repos/medicode/tensor2tensor/contributors","subscribers_url":"https://api.github.com/repos/medicode/tensor2tensor/subscribers","subscription_url":"https://api.github.com/repos/medicode/tensor2tensor/subscription","commits_url":"https://api.github.com/repos/medicode/tensor2tensor/commits{/sha}","git_commits_url":"https://api.github.com/repos/medicode/tensor2tensor/git/commits{/sha}","comments_url":"https://api.github.com/repos/medicode/tensor2tensor/comments{/number}","issue_comment_url":"https://api.github.com/repos/medicode/tensor2tensor/issues/comments{/number}","contents_url":"https://api.github.com/repos/medicode/tensor2tensor/contents/{+path}","compare_url":"https://api.github.com/repos/medicode/tensor2tensor/compare/{base}...{head}","merges_url":"https://api.github.com/repos/medicode/tensor2tensor/merges","archive_url":"https://api.github.com/repos/medicode/tensor2tensor/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/medicode/tensor2tensor/downloads","issues_url":"https://api.github.com/repos/medicode/tensor2tensor/issues{/number}","pulls_url":"https://api.github.com/repos/medicode/tensor2tensor/pulls{/number}","milestones_url":"https://api.github.com/repos/medicode/tensor2tensor/milestones{/number}","notifications_url":"https://api.github.com/repos/medicode/tensor2tensor/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/medicode/tensor2tensor/labels{/name}","releases_url":"https://api.github.com/repos/medicode/tensor2tensor/releases{/id}","deployments_url":"https://api.github.com/repos/medicode/tensor2tensor/deployments","created_at":"2017-10-01T21:04:04Z","updated_at":"2017-10-01T21:04:06Z","pushed_at":"2017-11-11T05:15:28Z","git_url":"git://github.com/medicode/tensor2tensor.git","ssh_url":"git@github.com:medicode/tensor2tensor.git","clone_url":"https://github.com/medicode/tensor2tensor.git","svn_url":"https://github.com/medicode/tensor2tensor","homepage":null,"size":1490,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/medicode/tensor2tensor/pulls/7"},"html":{"href":"https://github.com/medicode/tensor2tensor/pull/7"},"issue":{"href":"https://api.github.com/repos/medicode/tensor2tensor/issues/7"},"comments":{"href":"https://api.github.com/repos/medicode/tensor2tensor/issues/7/comments"},"review_comments":{"href":"https://api.github.com/repos/medicode/tensor2tensor/pulls/7/comments"},"review_comment":{"href":"https://api.github.com/repos/medicode/tensor2tensor/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/medicode/tensor2tensor/pulls/7/commits"},"statuses":{"href":"https://api.github.com/repos/medicode/tensor2tensor/statuses/2f99e2090bf62582d4ba914c919412a9a133dccd"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"epurdyf"}}