{"action":"created","comment":{"url":"https://api.github.com/repos/awslabs/sockeye/pulls/comments/121890245","pull_request_review_id":43952922,"id":121890245,"diff_hunk":"@@ -79,14 +87,41 @@ def main():\n     assert args.optimized_metric == C.BLEU or args.optimized_metric in args.metrics, \\\n         \"Must optimize either BLEU or one of tracked metrics (--metrics)\"\n \n+    # Checking status of output folder, resumption, etc.\n     output_folder = os.path.abspath(args.output)\n-    if not os.path.exists(output_folder):\n+    resume_training = False\n+    training_state_dir = os.path.join(output_folder, C.TRAINING_STATE_DIRNAME)\n+    if os.path.exists(output_folder):\n+        if args.overwrite_output:\n+            shutil.rmtree(output_folder)\n+            os.makedirs(output_folder)\n+        elif os.path.exists(training_state_dir):\n+            with open(os.path.join(output_folder, C.ARGS_STATE_NAME), \"r\") as fp:\n+                old_args = json.load(fp)\n+            arg_diffs = _dict_difference(vars(args), old_args) | _dict_difference(old_args, vars(args))\n+            # Remove args that may differ without affecting the training.\n+            # TODO: should we include device-ids and use-cpu?\n+            # Are there more?\n+            arg_diffs -= set([\"overwrite_output\", \"use-tensorboard\", \"quiet\", \"align_plot_prefix\", \"sure_align_threshold\"])","path":"sockeye/train.py","position":45,"original_position":45,"commit_id":"1885483e7e822ff8de4d0f56aa71d3cfffc8d81f","original_commit_id":"1885483e7e822ff8de4d0f56aa71d3cfffc8d81f","user":{"login":"fhieber","id":2027990,"avatar_url":"https://avatars1.githubusercontent.com/u/2027990?v=3","gravatar_id":"","url":"https://api.github.com/users/fhieber","html_url":"https://github.com/fhieber","followers_url":"https://api.github.com/users/fhieber/followers","following_url":"https://api.github.com/users/fhieber/following{/other_user}","gists_url":"https://api.github.com/users/fhieber/gists{/gist_id}","starred_url":"https://api.github.com/users/fhieber/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fhieber/subscriptions","organizations_url":"https://api.github.com/users/fhieber/orgs","repos_url":"https://api.github.com/users/fhieber/repos","events_url":"https://api.github.com/users/fhieber/events{/privacy}","received_events_url":"https://api.github.com/users/fhieber/received_events","type":"User","site_admin":false},"body":"lets define this set in constants.py","created_at":"2017-06-14T08:52:57Z","updated_at":"2017-06-14T08:54:26Z","html_url":"https://github.com/awslabs/sockeye/pull/7#discussion_r121890245","pull_request_url":"https://api.github.com/repos/awslabs/sockeye/pulls/7","_links":{"self":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/comments/121890245"},"html":{"href":"https://github.com/awslabs/sockeye/pull/7#discussion_r121890245"},"pull_request":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/7"}}},"pull_request":{"url":"https://api.github.com/repos/awslabs/sockeye/pulls/7","id":125359651,"html_url":"https://github.com/awslabs/sockeye/pull/7","diff_url":"https://github.com/awslabs/sockeye/pull/7.diff","patch_url":"https://github.com/awslabs/sockeye/pull/7.patch","issue_url":"https://api.github.com/repos/awslabs/sockeye/issues/7","number":7,"state":"open","locked":false,"title":"Support for continuation of aborted training","user":{"login":"davvil","id":683589,"avatar_url":"https://avatars3.githubusercontent.com/u/683589?v=3","gravatar_id":"","url":"https://api.github.com/users/davvil","html_url":"https://github.com/davvil","followers_url":"https://api.github.com/users/davvil/followers","following_url":"https://api.github.com/users/davvil/following{/other_user}","gists_url":"https://api.github.com/users/davvil/gists{/gist_id}","starred_url":"https://api.github.com/users/davvil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davvil/subscriptions","organizations_url":"https://api.github.com/users/davvil/orgs","repos_url":"https://api.github.com/users/davvil/repos","events_url":"https://api.github.com/users/davvil/events{/privacy}","received_events_url":"https://api.github.com/users/davvil/received_events","type":"User","site_admin":false},"body":"Transparently continue a training if it was aborted. **Important:** Start sockeye with exactly the same arguments.\r\n\r\nBecause of the asynchronous decoding for BLEU evaluation, some BLEU scores may get lost in the continuation, but this should not have a big effect on the final result.\r\n\r\nResults are not yet 100% equal in the continuation, but are certainly in the same range. There must be still some part of the \"traning state\" missing, put it is surely usable, especially if dealing with long trainings.","created_at":"2017-06-13T13:10:45Z","updated_at":"2017-06-14T08:54:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"63e93e664c259b3a7cde902b2a6ea1c431ce7634","assignee":null,"assignees":[],"requested_reviewers":[{"login":"tdomhan","id":363737,"avatar_url":"https://avatars2.githubusercontent.com/u/363737?v=3","gravatar_id":"","url":"https://api.github.com/users/tdomhan","html_url":"https://github.com/tdomhan","followers_url":"https://api.github.com/users/tdomhan/followers","following_url":"https://api.github.com/users/tdomhan/following{/other_user}","gists_url":"https://api.github.com/users/tdomhan/gists{/gist_id}","starred_url":"https://api.github.com/users/tdomhan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tdomhan/subscriptions","organizations_url":"https://api.github.com/users/tdomhan/orgs","repos_url":"https://api.github.com/users/tdomhan/repos","events_url":"https://api.github.com/users/tdomhan/events{/privacy}","received_events_url":"https://api.github.com/users/tdomhan/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/awslabs/sockeye/pulls/7/commits","review_comments_url":"https://api.github.com/repos/awslabs/sockeye/pulls/7/comments","review_comment_url":"https://api.github.com/repos/awslabs/sockeye/pulls/comments{/number}","comments_url":"https://api.github.com/repos/awslabs/sockeye/issues/7/comments","statuses_url":"https://api.github.com/repos/awslabs/sockeye/statuses/1885483e7e822ff8de4d0f56aa71d3cfffc8d81f","head":{"label":"awslabs:checkpoint","ref":"checkpoint","sha":"1885483e7e822ff8de4d0f56aa71d3cfffc8d81f","user":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars3.githubusercontent.com/u/3299148?v=3","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"repo":{"id":93721432,"name":"sockeye","full_name":"awslabs/sockeye","owner":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars3.githubusercontent.com/u/3299148?v=3","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/awslabs/sockeye","description":"Sequence-to-sequence framework with a focus on Neural Machine Translation based on MXNet","fork":false,"url":"https://api.github.com/repos/awslabs/sockeye","forks_url":"https://api.github.com/repos/awslabs/sockeye/forks","keys_url":"https://api.github.com/repos/awslabs/sockeye/keys{/key_id}","collaborators_url":"https://api.github.com/repos/awslabs/sockeye/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/awslabs/sockeye/teams","hooks_url":"https://api.github.com/repos/awslabs/sockeye/hooks","issue_events_url":"https://api.github.com/repos/awslabs/sockeye/issues/events{/number}","events_url":"https://api.github.com/repos/awslabs/sockeye/events","assignees_url":"https://api.github.com/repos/awslabs/sockeye/assignees{/user}","branches_url":"https://api.github.com/repos/awslabs/sockeye/branches{/branch}","tags_url":"https://api.github.com/repos/awslabs/sockeye/tags","blobs_url":"https://api.github.com/repos/awslabs/sockeye/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/awslabs/sockeye/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/awslabs/sockeye/git/refs{/sha}","trees_url":"https://api.github.com/repos/awslabs/sockeye/git/trees{/sha}","statuses_url":"https://api.github.com/repos/awslabs/sockeye/statuses/{sha}","languages_url":"https://api.github.com/repos/awslabs/sockeye/languages","stargazers_url":"https://api.github.com/repos/awslabs/sockeye/stargazers","contributors_url":"https://api.github.com/repos/awslabs/sockeye/contributors","subscribers_url":"https://api.github.com/repos/awslabs/sockeye/subscribers","subscription_url":"https://api.github.com/repos/awslabs/sockeye/subscription","commits_url":"https://api.github.com/repos/awslabs/sockeye/commits{/sha}","git_commits_url":"https://api.github.com/repos/awslabs/sockeye/git/commits{/sha}","comments_url":"https://api.github.com/repos/awslabs/sockeye/comments{/number}","issue_comment_url":"https://api.github.com/repos/awslabs/sockeye/issues/comments{/number}","contents_url":"https://api.github.com/repos/awslabs/sockeye/contents/{+path}","compare_url":"https://api.github.com/repos/awslabs/sockeye/compare/{base}...{head}","merges_url":"https://api.github.com/repos/awslabs/sockeye/merges","archive_url":"https://api.github.com/repos/awslabs/sockeye/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/awslabs/sockeye/downloads","issues_url":"https://api.github.com/repos/awslabs/sockeye/issues{/number}","pulls_url":"https://api.github.com/repos/awslabs/sockeye/pulls{/number}","milestones_url":"https://api.github.com/repos/awslabs/sockeye/milestones{/number}","notifications_url":"https://api.github.com/repos/awslabs/sockeye/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/awslabs/sockeye/labels{/name}","releases_url":"https://api.github.com/repos/awslabs/sockeye/releases{/id}","deployments_url":"https://api.github.com/repos/awslabs/sockeye/deployments","created_at":"2017-06-08T07:44:30Z","updated_at":"2017-06-14T07:32:12Z","pushed_at":"2017-06-14T08:42:33Z","git_url":"git://github.com/awslabs/sockeye.git","ssh_url":"git@github.com:awslabs/sockeye.git","clone_url":"https://github.com/awslabs/sockeye.git","svn_url":"https://github.com/awslabs/sockeye","homepage":"","size":141,"stargazers_count":63,"watchers_count":63,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":15,"mirror_url":null,"open_issues_count":1,"forks":15,"open_issues":1,"watchers":63,"default_branch":"master"}},"base":{"label":"awslabs:master","ref":"master","sha":"6b525be0c812228c5a8878aed56ab6137a34617f","user":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars3.githubusercontent.com/u/3299148?v=3","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"repo":{"id":93721432,"name":"sockeye","full_name":"awslabs/sockeye","owner":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars3.githubusercontent.com/u/3299148?v=3","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/awslabs/sockeye","description":"Sequence-to-sequence framework with a focus on Neural Machine Translation based on MXNet","fork":false,"url":"https://api.github.com/repos/awslabs/sockeye","forks_url":"https://api.github.com/repos/awslabs/sockeye/forks","keys_url":"https://api.github.com/repos/awslabs/sockeye/keys{/key_id}","collaborators_url":"https://api.github.com/repos/awslabs/sockeye/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/awslabs/sockeye/teams","hooks_url":"https://api.github.com/repos/awslabs/sockeye/hooks","issue_events_url":"https://api.github.com/repos/awslabs/sockeye/issues/events{/number}","events_url":"https://api.github.com/repos/awslabs/sockeye/events","assignees_url":"https://api.github.com/repos/awslabs/sockeye/assignees{/user}","branches_url":"https://api.github.com/repos/awslabs/sockeye/branches{/branch}","tags_url":"https://api.github.com/repos/awslabs/sockeye/tags","blobs_url":"https://api.github.com/repos/awslabs/sockeye/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/awslabs/sockeye/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/awslabs/sockeye/git/refs{/sha}","trees_url":"https://api.github.com/repos/awslabs/sockeye/git/trees{/sha}","statuses_url":"https://api.github.com/repos/awslabs/sockeye/statuses/{sha}","languages_url":"https://api.github.com/repos/awslabs/sockeye/languages","stargazers_url":"https://api.github.com/repos/awslabs/sockeye/stargazers","contributors_url":"https://api.github.com/repos/awslabs/sockeye/contributors","subscribers_url":"https://api.github.com/repos/awslabs/sockeye/subscribers","subscription_url":"https://api.github.com/repos/awslabs/sockeye/subscription","commits_url":"https://api.github.com/repos/awslabs/sockeye/commits{/sha}","git_commits_url":"https://api.github.com/repos/awslabs/sockeye/git/commits{/sha}","comments_url":"https://api.github.com/repos/awslabs/sockeye/comments{/number}","issue_comment_url":"https://api.github.com/repos/awslabs/sockeye/issues/comments{/number}","contents_url":"https://api.github.com/repos/awslabs/sockeye/contents/{+path}","compare_url":"https://api.github.com/repos/awslabs/sockeye/compare/{base}...{head}","merges_url":"https://api.github.com/repos/awslabs/sockeye/merges","archive_url":"https://api.github.com/repos/awslabs/sockeye/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/awslabs/sockeye/downloads","issues_url":"https://api.github.com/repos/awslabs/sockeye/issues{/number}","pulls_url":"https://api.github.com/repos/awslabs/sockeye/pulls{/number}","milestones_url":"https://api.github.com/repos/awslabs/sockeye/milestones{/number}","notifications_url":"https://api.github.com/repos/awslabs/sockeye/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/awslabs/sockeye/labels{/name}","releases_url":"https://api.github.com/repos/awslabs/sockeye/releases{/id}","deployments_url":"https://api.github.com/repos/awslabs/sockeye/deployments","created_at":"2017-06-08T07:44:30Z","updated_at":"2017-06-14T07:32:12Z","pushed_at":"2017-06-14T08:42:33Z","git_url":"git://github.com/awslabs/sockeye.git","ssh_url":"git@github.com:awslabs/sockeye.git","clone_url":"https://github.com/awslabs/sockeye.git","svn_url":"https://github.com/awslabs/sockeye","homepage":"","size":141,"stargazers_count":63,"watchers_count":63,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":15,"mirror_url":null,"open_issues_count":1,"forks":15,"open_issues":1,"watchers":63,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/7"},"html":{"href":"https://github.com/awslabs/sockeye/pull/7"},"issue":{"href":"https://api.github.com/repos/awslabs/sockeye/issues/7"},"comments":{"href":"https://api.github.com/repos/awslabs/sockeye/issues/7/comments"},"review_comments":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/7/comments"},"review_comment":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/7/commits"},"statuses":{"href":"https://api.github.com/repos/awslabs/sockeye/statuses/1885483e7e822ff8de4d0f56aa71d3cfffc8d81f"}}}}
{"actor":{"display_login":"fhieber"}}