{"action":"created","comment":{"url":"https://api.github.com/repos/Yelp/paasta/pulls/comments/144141385","pull_request_review_id":68757986,"id":144141385,"diff_hunk":"@@ -234,53 +255,54 @@ def wait_and_terminate(self, slave, drain_timeout, dry_run, region=None, should_\n         :param region to connect to ec2\n         :param dry_run: Don't drain or make changes to spot fleet if True\"\"\"\n         ec2_client = boto3.client('ec2', region_name=region)\n+        self.log.info(\"Starting TERMINATING: {} (Hostname = {}, IP = {})\".format(\n+            slave.instance_id,\n+            slave.hostname,\n+            slave.ip,\n+        ))\n         try:\n             # This loop should always finish because the maintenance window should trigger is_ready_to_kill\n             # being true. Just in case though we set a timeout and terminate anyway\n-            with Timeout(seconds=drain_timeout + 300):\n-                while True:\n-                    instance_id = slave.instance_id\n-                    if not instance_id:\n-                        self.log.warning(\n-                            \"Didn't find instance ID for slave: {}. Skipping terminating\".format(slave.pid),\n-                        )\n-                        continue\n-                    # Check if no tasks are running or we have reached the maintenance window\n-                    if not should_drain or is_safe_to_kill(slave.hostname):\n-                        if not should_drain and not dry_run:\n-                            self.log.info(\"Draining is disabled, just waiting 5 minutes!\")\n-                            time.sleep(300)\n-                        self.log.info(\"TERMINATING: {} (Hostname = {}, IP = {})\".format(\n-                            instance_id,\n-                            slave.hostname,\n-                            slave.ip,\n-                        ))\n-                        try:\n-                            ec2_client.terminate_instances(InstanceIds=[instance_id], DryRun=dry_run)\n-                        except ClientError as e:\n-                            if e.response['Error'].get('Code') == 'DryRunOperation':\n-                                pass\n-                            else:\n-                                raise\n-                        break\n-                    else:\n-                        self.log.info(\"Instance {}: NOT ready to kill\".format(instance_id))\n-                    self.log.debug(\"Waiting 5 seconds and then checking again\")\n-                    time.sleep(5)\n+            while True:\n+                instance_id = slave.instance_id\n+                if not instance_id:\n+                    self.log.warning(\n+                        \"Didn't find instance ID for slave: {}. Skipping terminating\".format(slave.pid),\n+                    )\n+                    continue\n+                # Check if no tasks are running or we have reached the maintenance window\n+                if self.can_kill(slave.hostname, should_drain, dry_run, timer):\n+                    self.log.info(\"TERMINATING: {} (Hostname = {}, IP = {})\".format(\n+                        instance_id,\n+                        slave.hostname,\n+                        slave.ip,\n+                    ))\n+                    try:\n+                        ec2_client.terminate_instances(InstanceIds=[instance_id], DryRun=dry_run)\n+                    except ClientError as e:\n+                        if e.response['Error'].get('Code') == 'DryRunOperation':\n+                            pass\n+                        else:\n+                            raise\n+                    break\n+                else:\n+                    self.log.info(\"Instance {}: NOT ready to kill\".format(instance_id))\n+                self.log.debug(\"Waiting 5 seconds and then checking again\")\n+                await asyncio.sleep(5)\n         except TimeoutError:\n             self.log.error(\"Timed out after {} waiting to drain {}, now terminating anyway\".format(\n                 drain_timeout,\n                 slave.pid,\n             ))\n             try:\n-                ec2_client.terminate_instances(InstanceIds=instance_id, DryRun=dry_run)\n+                ec2_client.terminate_instances(InstanceIds=[instance_id], DryRun=dry_run)\n             except ClientError as e:\n                 if e.response['Error'].get('Code') == 'DryRunOperation':\n                     pass\n                 else:\n                     raise\n \n-    def scale_resource(self, current_capacity, target_capacity):\n+    async def scale_resource(self, current_capacity, target_capacity):","path":"paasta_tools/autoscaling/autoscaling_cluster_lib.py","position":205,"original_position":205,"commit_id":"5be0f36154d518455963e6b6e4aef4b1ea86e09e","original_commit_id":"5be0f36154d518455963e6b6e4aef4b1ea86e09e","user":{"login":"matthewbentley","id":1930163,"avatar_url":"https://avatars1.githubusercontent.com/u/1930163?v=4","gravatar_id":"","url":"https://api.github.com/users/matthewbentley","html_url":"https://github.com/matthewbentley","followers_url":"https://api.github.com/users/matthewbentley/followers","following_url":"https://api.github.com/users/matthewbentley/following{/other_user}","gists_url":"https://api.github.com/users/matthewbentley/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewbentley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewbentley/subscriptions","organizations_url":"https://api.github.com/users/matthewbentley/orgs","repos_url":"https://api.github.com/users/matthewbentley/repos","events_url":"https://api.github.com/users/matthewbentley/events{/privacy}","received_events_url":"https://api.github.com/users/matthewbentley/received_events","type":"User","site_admin":false},"body":"This function needs an `await` call for the branches other than `delta < 0`","created_at":"2017-10-11T21:29:05Z","updated_at":"2017-10-11T21:34:33Z","html_url":"https://github.com/Yelp/paasta/pull/1496#discussion_r144141385","pull_request_url":"https://api.github.com/repos/Yelp/paasta/pulls/1496","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/comments/144141385"},"html":{"href":"https://github.com/Yelp/paasta/pull/1496#discussion_r144141385"},"pull_request":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/1496"}}},"pull_request":{"url":"https://api.github.com/repos/Yelp/paasta/pulls/1496","id":139814712,"html_url":"https://github.com/Yelp/paasta/pull/1496","diff_url":"https://github.com/Yelp/paasta/pull/1496.diff","patch_url":"https://github.com/Yelp/paasta/pull/1496.patch","issue_url":"https://api.github.com/repos/Yelp/paasta/issues/1496","number":1496,"state":"open","locked":false,"title":"Async scale down cluster resource (ie parallel draining+killing of slaves in an sfr/asg)","user":{"login":"matthewbentley","id":1930163,"avatar_url":"https://avatars1.githubusercontent.com/u/1930163?v=4","gravatar_id":"","url":"https://api.github.com/users/matthewbentley","html_url":"https://github.com/matthewbentley","followers_url":"https://api.github.com/users/matthewbentley/followers","following_url":"https://api.github.com/users/matthewbentley/following{/other_user}","gists_url":"https://api.github.com/users/matthewbentley/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewbentley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewbentley/subscriptions","organizations_url":"https://api.github.com/users/matthewbentley/orgs","repos_url":"https://api.github.com/users/matthewbentley/repos","events_url":"https://api.github.com/users/matthewbentley/events{/privacy}","received_events_url":"https://api.github.com/users/matthewbentley/received_events","type":"User","site_admin":false},"body":"This changes the cluster autosclaler so that within one resource, it will attempt to kill multiple slaves at once.\r\n\r\nBasically, it creates a coroutine per slave to kill, and then runs that until it hits `await asyncio.sleep(5)`.  At which point the execution will be suspended, and the `downscale_aws_resource` will resume, going on to the next slave.  At the end of `downscale_aws_resource`, we wait until all coroutines have completed (or been canceled) before returning.\r\n\r\nThe logic in `downscale_aws_resource` is a bit iffy, but I'm confident in the proper interactions between having multiple `gracefully_terminate_slave`s and `wait_and_terminate`s running at at time.\r\n\r\nThis is not actually multithreaded, as only one coroutine will run at once, but there is a bit of difficulty around setting the capacity correctly (and then resetting in case of failure).\r\n\r\nAddresses (but not totally) #1495","created_at":"2017-09-07T14:07:21Z","updated_at":"2017-10-11T21:34:33Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d909c77a3672f7b034476322c3479239244fac95","assignee":{"login":"matthewbentley","id":1930163,"avatar_url":"https://avatars1.githubusercontent.com/u/1930163?v=4","gravatar_id":"","url":"https://api.github.com/users/matthewbentley","html_url":"https://github.com/matthewbentley","followers_url":"https://api.github.com/users/matthewbentley/followers","following_url":"https://api.github.com/users/matthewbentley/following{/other_user}","gists_url":"https://api.github.com/users/matthewbentley/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewbentley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewbentley/subscriptions","organizations_url":"https://api.github.com/users/matthewbentley/orgs","repos_url":"https://api.github.com/users/matthewbentley/repos","events_url":"https://api.github.com/users/matthewbentley/events{/privacy}","received_events_url":"https://api.github.com/users/matthewbentley/received_events","type":"User","site_admin":false},"assignees":[{"login":"matthewbentley","id":1930163,"avatar_url":"https://avatars1.githubusercontent.com/u/1930163?v=4","gravatar_id":"","url":"https://api.github.com/users/matthewbentley","html_url":"https://github.com/matthewbentley","followers_url":"https://api.github.com/users/matthewbentley/followers","following_url":"https://api.github.com/users/matthewbentley/following{/other_user}","gists_url":"https://api.github.com/users/matthewbentley/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewbentley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewbentley/subscriptions","organizations_url":"https://api.github.com/users/matthewbentley/orgs","repos_url":"https://api.github.com/users/matthewbentley/repos","events_url":"https://api.github.com/users/matthewbentley/events{/privacy}","received_events_url":"https://api.github.com/users/matthewbentley/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/Yelp/paasta/pulls/1496/commits","review_comments_url":"https://api.github.com/repos/Yelp/paasta/pulls/1496/comments","review_comment_url":"https://api.github.com/repos/Yelp/paasta/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Yelp/paasta/issues/1496/comments","statuses_url":"https://api.github.com/repos/Yelp/paasta/statuses/5be0f36154d518455963e6b6e4aef4b1ea86e09e","head":{"label":"Yelp:async-scale-down-cluster","ref":"async-scale-down-cluster","sha":"5be0f36154d518455963e6b6e4aef4b1ea86e09e","user":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"repo":{"id":44998824,"name":"paasta","full_name":"Yelp/paasta","owner":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Yelp/paasta","description":"An open, distributed platform as a service","fork":false,"url":"https://api.github.com/repos/Yelp/paasta","forks_url":"https://api.github.com/repos/Yelp/paasta/forks","keys_url":"https://api.github.com/repos/Yelp/paasta/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Yelp/paasta/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Yelp/paasta/teams","hooks_url":"https://api.github.com/repos/Yelp/paasta/hooks","issue_events_url":"https://api.github.com/repos/Yelp/paasta/issues/events{/number}","events_url":"https://api.github.com/repos/Yelp/paasta/events","assignees_url":"https://api.github.com/repos/Yelp/paasta/assignees{/user}","branches_url":"https://api.github.com/repos/Yelp/paasta/branches{/branch}","tags_url":"https://api.github.com/repos/Yelp/paasta/tags","blobs_url":"https://api.github.com/repos/Yelp/paasta/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Yelp/paasta/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Yelp/paasta/git/refs{/sha}","trees_url":"https://api.github.com/repos/Yelp/paasta/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Yelp/paasta/statuses/{sha}","languages_url":"https://api.github.com/repos/Yelp/paasta/languages","stargazers_url":"https://api.github.com/repos/Yelp/paasta/stargazers","contributors_url":"https://api.github.com/repos/Yelp/paasta/contributors","subscribers_url":"https://api.github.com/repos/Yelp/paasta/subscribers","subscription_url":"https://api.github.com/repos/Yelp/paasta/subscription","commits_url":"https://api.github.com/repos/Yelp/paasta/commits{/sha}","git_commits_url":"https://api.github.com/repos/Yelp/paasta/git/commits{/sha}","comments_url":"https://api.github.com/repos/Yelp/paasta/comments{/number}","issue_comment_url":"https://api.github.com/repos/Yelp/paasta/issues/comments{/number}","contents_url":"https://api.github.com/repos/Yelp/paasta/contents/{+path}","compare_url":"https://api.github.com/repos/Yelp/paasta/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Yelp/paasta/merges","archive_url":"https://api.github.com/repos/Yelp/paasta/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Yelp/paasta/downloads","issues_url":"https://api.github.com/repos/Yelp/paasta/issues{/number}","pulls_url":"https://api.github.com/repos/Yelp/paasta/pulls{/number}","milestones_url":"https://api.github.com/repos/Yelp/paasta/milestones{/number}","notifications_url":"https://api.github.com/repos/Yelp/paasta/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Yelp/paasta/labels{/name}","releases_url":"https://api.github.com/repos/Yelp/paasta/releases{/id}","deployments_url":"https://api.github.com/repos/Yelp/paasta/deployments","created_at":"2015-10-26T21:35:53Z","updated_at":"2017-10-11T04:27:39Z","pushed_at":"2017-10-11T20:47:34Z","git_url":"git://github.com/Yelp/paasta.git","ssh_url":"git@github.com:Yelp/paasta.git","clone_url":"https://github.com/Yelp/paasta.git","svn_url":"https://github.com/Yelp/paasta","homepage":"","size":54062,"stargazers_count":1127,"watchers_count":1127,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":138,"mirror_url":null,"open_issues_count":100,"forks":138,"open_issues":100,"watchers":1127,"default_branch":"master"}},"base":{"label":"Yelp:master","ref":"master","sha":"d8301cd5466b44972e98968361771d8a6ca87d66","user":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"repo":{"id":44998824,"name":"paasta","full_name":"Yelp/paasta","owner":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Yelp/paasta","description":"An open, distributed platform as a service","fork":false,"url":"https://api.github.com/repos/Yelp/paasta","forks_url":"https://api.github.com/repos/Yelp/paasta/forks","keys_url":"https://api.github.com/repos/Yelp/paasta/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Yelp/paasta/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Yelp/paasta/teams","hooks_url":"https://api.github.com/repos/Yelp/paasta/hooks","issue_events_url":"https://api.github.com/repos/Yelp/paasta/issues/events{/number}","events_url":"https://api.github.com/repos/Yelp/paasta/events","assignees_url":"https://api.github.com/repos/Yelp/paasta/assignees{/user}","branches_url":"https://api.github.com/repos/Yelp/paasta/branches{/branch}","tags_url":"https://api.github.com/repos/Yelp/paasta/tags","blobs_url":"https://api.github.com/repos/Yelp/paasta/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Yelp/paasta/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Yelp/paasta/git/refs{/sha}","trees_url":"https://api.github.com/repos/Yelp/paasta/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Yelp/paasta/statuses/{sha}","languages_url":"https://api.github.com/repos/Yelp/paasta/languages","stargazers_url":"https://api.github.com/repos/Yelp/paasta/stargazers","contributors_url":"https://api.github.com/repos/Yelp/paasta/contributors","subscribers_url":"https://api.github.com/repos/Yelp/paasta/subscribers","subscription_url":"https://api.github.com/repos/Yelp/paasta/subscription","commits_url":"https://api.github.com/repos/Yelp/paasta/commits{/sha}","git_commits_url":"https://api.github.com/repos/Yelp/paasta/git/commits{/sha}","comments_url":"https://api.github.com/repos/Yelp/paasta/comments{/number}","issue_comment_url":"https://api.github.com/repos/Yelp/paasta/issues/comments{/number}","contents_url":"https://api.github.com/repos/Yelp/paasta/contents/{+path}","compare_url":"https://api.github.com/repos/Yelp/paasta/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Yelp/paasta/merges","archive_url":"https://api.github.com/repos/Yelp/paasta/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Yelp/paasta/downloads","issues_url":"https://api.github.com/repos/Yelp/paasta/issues{/number}","pulls_url":"https://api.github.com/repos/Yelp/paasta/pulls{/number}","milestones_url":"https://api.github.com/repos/Yelp/paasta/milestones{/number}","notifications_url":"https://api.github.com/repos/Yelp/paasta/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Yelp/paasta/labels{/name}","releases_url":"https://api.github.com/repos/Yelp/paasta/releases{/id}","deployments_url":"https://api.github.com/repos/Yelp/paasta/deployments","created_at":"2015-10-26T21:35:53Z","updated_at":"2017-10-11T04:27:39Z","pushed_at":"2017-10-11T20:47:34Z","git_url":"git://github.com/Yelp/paasta.git","ssh_url":"git@github.com:Yelp/paasta.git","clone_url":"https://github.com/Yelp/paasta.git","svn_url":"https://github.com/Yelp/paasta","homepage":"","size":54062,"stargazers_count":1127,"watchers_count":1127,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":138,"mirror_url":null,"open_issues_count":100,"forks":138,"open_issues":100,"watchers":1127,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/1496"},"html":{"href":"https://github.com/Yelp/paasta/pull/1496"},"issue":{"href":"https://api.github.com/repos/Yelp/paasta/issues/1496"},"comments":{"href":"https://api.github.com/repos/Yelp/paasta/issues/1496/comments"},"review_comments":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/1496/comments"},"review_comment":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/1496/commits"},"statuses":{"href":"https://api.github.com/repos/Yelp/paasta/statuses/5be0f36154d518455963e6b6e4aef4b1ea86e09e"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"matthewbentley"}}