{"action":"created","comment":{"url":"https://api.github.com/repos/Yelp/task_processing/pulls/comments/133211397","pull_request_review_id":56364476,"id":133211397,"diff_hunk":"@@ -542,12 +545,30 @@ def statusUpdate(self, driver, update):\n         if task_id not in self.task_metadata:\n             # We assume that a terminal status update has been\n             # received for this task already.\n-            log.info('Ignoring this status update because a terminal status \\\n-                update has been receiced for this task already.')\n+            log.info('Ignoring this status update because a terminal status'\n+                     'update has been receiced for this task already.')\n             driver.acknowledgeStatusUpdate(update)\n             return\n+\n         md = self.task_metadata[task_id]\n \n+        # If we attempt to accept an offer that has been invalidated by master\n+        # for some reason such as offer has been rescinded or we have exceeded\n+        # offer_timeout, then we will get TASK_LOST status update back from\n+        # mesos master.\n+        if task_state == 'TASK_LOST' and \\\n+                'REASON_INVALID_OFFERS' == str(update.reason):\n+            # This task has not been launched. Therefore, we are going to\n+            # reenqueue it. We are not propogating any event up to the\n+            # application.\n+            log.warning('Received TASK_LOST from mesos master because we'\n+                        'attemted to accept an invalid offer.')\n+            self.task_metadata = self.task_metadata.discard(task_id)\n+            self.enqueue_task(md.task_config)","path":"task_processing/plugins/mesos/execution_framework.py","position":52,"original_position":52,"commit_id":"b79ab1475351d40f132fa830f3cff3e2cdd42227","original_commit_id":"b79ab1475351d40f132fa830f3cff3e2cdd42227","user":{"login":"Rob-Johnson","id":1389671,"avatar_url":"https://avatars1.githubusercontent.com/u/1389671?v=4","gravatar_id":"","url":"https://api.github.com/users/Rob-Johnson","html_url":"https://github.com/Rob-Johnson","followers_url":"https://api.github.com/users/Rob-Johnson/followers","following_url":"https://api.github.com/users/Rob-Johnson/following{/other_user}","gists_url":"https://api.github.com/users/Rob-Johnson/gists{/gist_id}","starred_url":"https://api.github.com/users/Rob-Johnson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Rob-Johnson/subscriptions","organizations_url":"https://api.github.com/users/Rob-Johnson/orgs","repos_url":"https://api.github.com/users/Rob-Johnson/repos","events_url":"https://api.github.com/users/Rob-Johnson/events{/privacy}","received_events_url":"https://api.github.com/users/Rob-Johnson/received_events","type":"User","site_admin":false},"body":"I think I agree - lets be consistent with our use of the executor and not make exceptions. We can make a 'default_mesos_stack' that includes all those executors we deem to be sensible - there is no reason to force everyone to do this.","created_at":"2017-08-15T14:57:07Z","updated_at":"2017-08-15T14:57:15Z","html_url":"https://github.com/Yelp/task_processing/pull/55#discussion_r133211397","pull_request_url":"https://api.github.com/repos/Yelp/task_processing/pulls/55","_links":{"self":{"href":"https://api.github.com/repos/Yelp/task_processing/pulls/comments/133211397"},"html":{"href":"https://github.com/Yelp/task_processing/pull/55#discussion_r133211397"},"pull_request":{"href":"https://api.github.com/repos/Yelp/task_processing/pulls/55"}}},"pull_request":{"url":"https://api.github.com/repos/Yelp/task_processing/pulls/55","id":135739583,"html_url":"https://github.com/Yelp/task_processing/pull/55","diff_url":"https://github.com/Yelp/task_processing/pull/55.diff","patch_url":"https://github.com/Yelp/task_processing/pull/55.patch","issue_url":"https://api.github.com/repos/Yelp/task_processing/issues/55","number":55,"state":"open","locked":false,"title":"Deal with TASK_LOST due to invalid offers","user":{"login":"sagar8192","id":5456218,"avatar_url":"https://avatars2.githubusercontent.com/u/5456218?v=4","gravatar_id":"","url":"https://api.github.com/users/sagar8192","html_url":"https://github.com/sagar8192","followers_url":"https://api.github.com/users/sagar8192/followers","following_url":"https://api.github.com/users/sagar8192/following{/other_user}","gists_url":"https://api.github.com/users/sagar8192/gists{/gist_id}","starred_url":"https://api.github.com/users/sagar8192/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sagar8192/subscriptions","organizations_url":"https://api.github.com/users/sagar8192/orgs","repos_url":"https://api.github.com/users/sagar8192/repos","events_url":"https://api.github.com/users/sagar8192/events{/privacy}","received_events_url":"https://api.github.com/users/sagar8192/received_events","type":"User","site_admin":false},"body":"I think it is better to re-enqueue the task in execution_framework than doing it in retrying_executor because not all applications have to use retrying_executor. This is something that execution framework should be able to deal with.","created_at":"2017-08-15T05:58:53Z","updated_at":"2017-08-15T14:57:15Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a518e771a057923879c3a56e5aa5ff2d9999b448","assignee":null,"assignees":[],"requested_reviewers":[{"login":"ssk2","id":355705,"avatar_url":"https://avatars1.githubusercontent.com/u/355705?v=4","gravatar_id":"","url":"https://api.github.com/users/ssk2","html_url":"https://github.com/ssk2","followers_url":"https://api.github.com/users/ssk2/followers","following_url":"https://api.github.com/users/ssk2/following{/other_user}","gists_url":"https://api.github.com/users/ssk2/gists{/gist_id}","starred_url":"https://api.github.com/users/ssk2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ssk2/subscriptions","organizations_url":"https://api.github.com/users/ssk2/orgs","repos_url":"https://api.github.com/users/ssk2/repos","events_url":"https://api.github.com/users/ssk2/events{/privacy}","received_events_url":"https://api.github.com/users/ssk2/received_events","type":"User","site_admin":false},{"login":"huadongliu","id":2797280,"avatar_url":"https://avatars1.githubusercontent.com/u/2797280?v=4","gravatar_id":"","url":"https://api.github.com/users/huadongliu","html_url":"https://github.com/huadongliu","followers_url":"https://api.github.com/users/huadongliu/followers","following_url":"https://api.github.com/users/huadongliu/following{/other_user}","gists_url":"https://api.github.com/users/huadongliu/gists{/gist_id}","starred_url":"https://api.github.com/users/huadongliu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/huadongliu/subscriptions","organizations_url":"https://api.github.com/users/huadongliu/orgs","repos_url":"https://api.github.com/users/huadongliu/repos","events_url":"https://api.github.com/users/huadongliu/events{/privacy}","received_events_url":"https://api.github.com/users/huadongliu/received_events","type":"User","site_admin":false},{"login":"charleskwwan","id":17325661,"avatar_url":"https://avatars2.githubusercontent.com/u/17325661?v=4","gravatar_id":"","url":"https://api.github.com/users/charleskwwan","html_url":"https://github.com/charleskwwan","followers_url":"https://api.github.com/users/charleskwwan/followers","following_url":"https://api.github.com/users/charleskwwan/following{/other_user}","gists_url":"https://api.github.com/users/charleskwwan/gists{/gist_id}","starred_url":"https://api.github.com/users/charleskwwan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/charleskwwan/subscriptions","organizations_url":"https://api.github.com/users/charleskwwan/orgs","repos_url":"https://api.github.com/users/charleskwwan/repos","events_url":"https://api.github.com/users/charleskwwan/events{/privacy}","received_events_url":"https://api.github.com/users/charleskwwan/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/Yelp/task_processing/pulls/55/commits","review_comments_url":"https://api.github.com/repos/Yelp/task_processing/pulls/55/comments","review_comment_url":"https://api.github.com/repos/Yelp/task_processing/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Yelp/task_processing/issues/55/comments","statuses_url":"https://api.github.com/repos/Yelp/task_processing/statuses/b79ab1475351d40f132fa830f3cff3e2cdd42227","head":{"label":"Yelp:sagarp-TASLPROC-87","ref":"sagarp-TASLPROC-87","sha":"b79ab1475351d40f132fa830f3cff3e2cdd42227","user":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"repo":{"id":91122217,"name":"task_processing","full_name":"Yelp/task_processing","owner":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Yelp/task_processing","description":"Interfaces and shared infrastructure for generic task processing @ Yelp","fork":false,"url":"https://api.github.com/repos/Yelp/task_processing","forks_url":"https://api.github.com/repos/Yelp/task_processing/forks","keys_url":"https://api.github.com/repos/Yelp/task_processing/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Yelp/task_processing/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Yelp/task_processing/teams","hooks_url":"https://api.github.com/repos/Yelp/task_processing/hooks","issue_events_url":"https://api.github.com/repos/Yelp/task_processing/issues/events{/number}","events_url":"https://api.github.com/repos/Yelp/task_processing/events","assignees_url":"https://api.github.com/repos/Yelp/task_processing/assignees{/user}","branches_url":"https://api.github.com/repos/Yelp/task_processing/branches{/branch}","tags_url":"https://api.github.com/repos/Yelp/task_processing/tags","blobs_url":"https://api.github.com/repos/Yelp/task_processing/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Yelp/task_processing/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Yelp/task_processing/git/refs{/sha}","trees_url":"https://api.github.com/repos/Yelp/task_processing/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Yelp/task_processing/statuses/{sha}","languages_url":"https://api.github.com/repos/Yelp/task_processing/languages","stargazers_url":"https://api.github.com/repos/Yelp/task_processing/stargazers","contributors_url":"https://api.github.com/repos/Yelp/task_processing/contributors","subscribers_url":"https://api.github.com/repos/Yelp/task_processing/subscribers","subscription_url":"https://api.github.com/repos/Yelp/task_processing/subscription","commits_url":"https://api.github.com/repos/Yelp/task_processing/commits{/sha}","git_commits_url":"https://api.github.com/repos/Yelp/task_processing/git/commits{/sha}","comments_url":"https://api.github.com/repos/Yelp/task_processing/comments{/number}","issue_comment_url":"https://api.github.com/repos/Yelp/task_processing/issues/comments{/number}","contents_url":"https://api.github.com/repos/Yelp/task_processing/contents/{+path}","compare_url":"https://api.github.com/repos/Yelp/task_processing/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Yelp/task_processing/merges","archive_url":"https://api.github.com/repos/Yelp/task_processing/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Yelp/task_processing/downloads","issues_url":"https://api.github.com/repos/Yelp/task_processing/issues{/number}","pulls_url":"https://api.github.com/repos/Yelp/task_processing/pulls{/number}","milestones_url":"https://api.github.com/repos/Yelp/task_processing/milestones{/number}","notifications_url":"https://api.github.com/repos/Yelp/task_processing/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Yelp/task_processing/labels{/name}","releases_url":"https://api.github.com/repos/Yelp/task_processing/releases{/id}","deployments_url":"https://api.github.com/repos/Yelp/task_processing/deployments","created_at":"2017-05-12T19:11:36Z","updated_at":"2017-06-15T14:38:17Z","pushed_at":"2017-08-15T14:51:27Z","git_url":"git://github.com/Yelp/task_processing.git","ssh_url":"git@github.com:Yelp/task_processing.git","clone_url":"https://github.com/Yelp/task_processing.git","svn_url":"https://github.com/Yelp/task_processing","homepage":null,"size":37411,"stargazers_count":5,"watchers_count":5,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":5,"forks":0,"open_issues":5,"watchers":5,"default_branch":"master"}},"base":{"label":"Yelp:master","ref":"master","sha":"b0bb4968542d530d98e5d79171924db18ac625e5","user":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"repo":{"id":91122217,"name":"task_processing","full_name":"Yelp/task_processing","owner":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Yelp/task_processing","description":"Interfaces and shared infrastructure for generic task processing @ Yelp","fork":false,"url":"https://api.github.com/repos/Yelp/task_processing","forks_url":"https://api.github.com/repos/Yelp/task_processing/forks","keys_url":"https://api.github.com/repos/Yelp/task_processing/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Yelp/task_processing/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Yelp/task_processing/teams","hooks_url":"https://api.github.com/repos/Yelp/task_processing/hooks","issue_events_url":"https://api.github.com/repos/Yelp/task_processing/issues/events{/number}","events_url":"https://api.github.com/repos/Yelp/task_processing/events","assignees_url":"https://api.github.com/repos/Yelp/task_processing/assignees{/user}","branches_url":"https://api.github.com/repos/Yelp/task_processing/branches{/branch}","tags_url":"https://api.github.com/repos/Yelp/task_processing/tags","blobs_url":"https://api.github.com/repos/Yelp/task_processing/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Yelp/task_processing/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Yelp/task_processing/git/refs{/sha}","trees_url":"https://api.github.com/repos/Yelp/task_processing/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Yelp/task_processing/statuses/{sha}","languages_url":"https://api.github.com/repos/Yelp/task_processing/languages","stargazers_url":"https://api.github.com/repos/Yelp/task_processing/stargazers","contributors_url":"https://api.github.com/repos/Yelp/task_processing/contributors","subscribers_url":"https://api.github.com/repos/Yelp/task_processing/subscribers","subscription_url":"https://api.github.com/repos/Yelp/task_processing/subscription","commits_url":"https://api.github.com/repos/Yelp/task_processing/commits{/sha}","git_commits_url":"https://api.github.com/repos/Yelp/task_processing/git/commits{/sha}","comments_url":"https://api.github.com/repos/Yelp/task_processing/comments{/number}","issue_comment_url":"https://api.github.com/repos/Yelp/task_processing/issues/comments{/number}","contents_url":"https://api.github.com/repos/Yelp/task_processing/contents/{+path}","compare_url":"https://api.github.com/repos/Yelp/task_processing/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Yelp/task_processing/merges","archive_url":"https://api.github.com/repos/Yelp/task_processing/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Yelp/task_processing/downloads","issues_url":"https://api.github.com/repos/Yelp/task_processing/issues{/number}","pulls_url":"https://api.github.com/repos/Yelp/task_processing/pulls{/number}","milestones_url":"https://api.github.com/repos/Yelp/task_processing/milestones{/number}","notifications_url":"https://api.github.com/repos/Yelp/task_processing/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Yelp/task_processing/labels{/name}","releases_url":"https://api.github.com/repos/Yelp/task_processing/releases{/id}","deployments_url":"https://api.github.com/repos/Yelp/task_processing/deployments","created_at":"2017-05-12T19:11:36Z","updated_at":"2017-06-15T14:38:17Z","pushed_at":"2017-08-15T14:51:27Z","git_url":"git://github.com/Yelp/task_processing.git","ssh_url":"git@github.com:Yelp/task_processing.git","clone_url":"https://github.com/Yelp/task_processing.git","svn_url":"https://github.com/Yelp/task_processing","homepage":null,"size":37411,"stargazers_count":5,"watchers_count":5,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":5,"forks":0,"open_issues":5,"watchers":5,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Yelp/task_processing/pulls/55"},"html":{"href":"https://github.com/Yelp/task_processing/pull/55"},"issue":{"href":"https://api.github.com/repos/Yelp/task_processing/issues/55"},"comments":{"href":"https://api.github.com/repos/Yelp/task_processing/issues/55/comments"},"review_comments":{"href":"https://api.github.com/repos/Yelp/task_processing/pulls/55/comments"},"review_comment":{"href":"https://api.github.com/repos/Yelp/task_processing/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Yelp/task_processing/pulls/55/commits"},"statuses":{"href":"https://api.github.com/repos/Yelp/task_processing/statuses/b79ab1475351d40f132fa830f3cff3e2cdd42227"}}}}
{"actor":{"display_login":"Rob-Johnson"}}