{"action":"created","comment":{"url":"https://api.github.com/repos/Yelp/paasta/pulls/comments/145189953","pull_request_review_id":69953136,"id":145189953,"diff_hunk":"@@ -207,80 +230,82 @@ def get_mesos_utilization_error(\n                 )\n                 raise ClusterAutoscalingError(error_message)\n \n-        region_pool_utilization_dict = get_resource_utilization_by_grouping(\n-            lambda slave: (slave['attributes']['pool'], slave['attributes']['datacenter'],),\n-            mesos_state,\n-        )[(self.resource['pool'], self.resource['region'],)]\n-\n-        self.log.debug(region_pool_utilization_dict)\n-        free_pool_resources = region_pool_utilization_dict['free']\n-        total_pool_resources = region_pool_utilization_dict['total']\n-        free_percs = []\n-        for pair in zip(free_pool_resources, total_pool_resources):\n-            free, total = pair[0], pair[1]\n-            if math.isclose(total, 0):\n-                continue\n-            free_percs.append(float(free) / float(total))\n-        utilization = 1.0 - min(free_percs)\n-        target_utilization = self.pool_settings.get('target_utilization', DEFAULT_TARGET_UTILIZATION)\n-        return utilization - target_utilization\n+    def can_kill(self, hostname, should_drain, dry_run, timer):\n+        if dry_run:\n+            return True\n+        if timer.ready():\n+            self.log.warning(\"Timer expired before slave ready to kill...\")","path":"paasta_tools/autoscaling/autoscaling_cluster_lib.py","position":105,"original_position":105,"commit_id":"9aa8d8d8dd300a49a01bd1694b22c7b09ea05a38","original_commit_id":"9aa8d8d8dd300a49a01bd1694b22c7b09ea05a38","user":{"login":"Rob-Johnson","id":1389671,"avatar_url":"https://avatars1.githubusercontent.com/u/1389671?v=4","gravatar_id":"","url":"https://api.github.com/users/Rob-Johnson","html_url":"https://github.com/Rob-Johnson","followers_url":"https://api.github.com/users/Rob-Johnson/followers","following_url":"https://api.github.com/users/Rob-Johnson/following{/other_user}","gists_url":"https://api.github.com/users/Rob-Johnson/gists{/gist_id}","starred_url":"https://api.github.com/users/Rob-Johnson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Rob-Johnson/subscriptions","organizations_url":"https://api.github.com/users/Rob-Johnson/orgs","repos_url":"https://api.github.com/users/Rob-Johnson/repos","events_url":"https://api.github.com/users/Rob-Johnson/events{/privacy}","received_events_url":"https://api.github.com/users/Rob-Johnson/received_events","type":"User","site_admin":false},"body":"can you hint at what this really means? is it a problem?","created_at":"2017-10-17T16:54:17Z","updated_at":"2017-10-17T17:01:43Z","html_url":"https://github.com/Yelp/paasta/pull/1496#discussion_r145189953","pull_request_url":"https://api.github.com/repos/Yelp/paasta/pulls/1496","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/comments/145189953"},"html":{"href":"https://github.com/Yelp/paasta/pull/1496#discussion_r145189953"},"pull_request":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/1496"}}},"pull_request":{"url":"https://api.github.com/repos/Yelp/paasta/pulls/1496","id":139814712,"html_url":"https://github.com/Yelp/paasta/pull/1496","diff_url":"https://github.com/Yelp/paasta/pull/1496.diff","patch_url":"https://github.com/Yelp/paasta/pull/1496.patch","issue_url":"https://api.github.com/repos/Yelp/paasta/issues/1496","number":1496,"state":"open","locked":false,"title":"Async scale down cluster resource (ie parallel draining+killing of slaves in an sfr/asg)","user":{"login":"matthewbentley","id":1930163,"avatar_url":"https://avatars1.githubusercontent.com/u/1930163?v=4","gravatar_id":"","url":"https://api.github.com/users/matthewbentley","html_url":"https://github.com/matthewbentley","followers_url":"https://api.github.com/users/matthewbentley/followers","following_url":"https://api.github.com/users/matthewbentley/following{/other_user}","gists_url":"https://api.github.com/users/matthewbentley/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewbentley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewbentley/subscriptions","organizations_url":"https://api.github.com/users/matthewbentley/orgs","repos_url":"https://api.github.com/users/matthewbentley/repos","events_url":"https://api.github.com/users/matthewbentley/events{/privacy}","received_events_url":"https://api.github.com/users/matthewbentley/received_events","type":"User","site_admin":false},"body":"This changes the cluster autosclaler so that within one resource, it will attempt to kill multiple slaves at once.\r\n\r\nBasically, it creates a coroutine per slave to kill, and then runs that until it hits `await asyncio.sleep(5)`.  At which point the execution will be suspended, and the `downscale_aws_resource` will resume, going on to the next slave.  At the end of `downscale_aws_resource`, we wait until all coroutines have completed (or been canceled) before returning.\r\n\r\nThe logic in `downscale_aws_resource` is a bit iffy, but I'm confident in the proper interactions between having multiple `gracefully_terminate_slave`s and `wait_and_terminate`s running at at time.\r\n\r\nThis is not actually multithreaded, as only one coroutine will run at once, but there is a bit of difficulty around setting the capacity correctly (and then resetting in case of failure).\r\n\r\nAddresses (but not totally) #1495","created_at":"2017-09-07T14:07:21Z","updated_at":"2017-10-17T17:01:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2bba6dd3857a94f702234a0b88f596e7b5109d1b","assignee":{"login":"matthewbentley","id":1930163,"avatar_url":"https://avatars1.githubusercontent.com/u/1930163?v=4","gravatar_id":"","url":"https://api.github.com/users/matthewbentley","html_url":"https://github.com/matthewbentley","followers_url":"https://api.github.com/users/matthewbentley/followers","following_url":"https://api.github.com/users/matthewbentley/following{/other_user}","gists_url":"https://api.github.com/users/matthewbentley/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewbentley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewbentley/subscriptions","organizations_url":"https://api.github.com/users/matthewbentley/orgs","repos_url":"https://api.github.com/users/matthewbentley/repos","events_url":"https://api.github.com/users/matthewbentley/events{/privacy}","received_events_url":"https://api.github.com/users/matthewbentley/received_events","type":"User","site_admin":false},"assignees":[{"login":"matthewbentley","id":1930163,"avatar_url":"https://avatars1.githubusercontent.com/u/1930163?v=4","gravatar_id":"","url":"https://api.github.com/users/matthewbentley","html_url":"https://github.com/matthewbentley","followers_url":"https://api.github.com/users/matthewbentley/followers","following_url":"https://api.github.com/users/matthewbentley/following{/other_user}","gists_url":"https://api.github.com/users/matthewbentley/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewbentley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewbentley/subscriptions","organizations_url":"https://api.github.com/users/matthewbentley/orgs","repos_url":"https://api.github.com/users/matthewbentley/repos","events_url":"https://api.github.com/users/matthewbentley/events{/privacy}","received_events_url":"https://api.github.com/users/matthewbentley/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/Yelp/paasta/pulls/1496/commits","review_comments_url":"https://api.github.com/repos/Yelp/paasta/pulls/1496/comments","review_comment_url":"https://api.github.com/repos/Yelp/paasta/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Yelp/paasta/issues/1496/comments","statuses_url":"https://api.github.com/repos/Yelp/paasta/statuses/9aa8d8d8dd300a49a01bd1694b22c7b09ea05a38","head":{"label":"Yelp:async-scale-down-cluster","ref":"async-scale-down-cluster","sha":"9aa8d8d8dd300a49a01bd1694b22c7b09ea05a38","user":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"repo":{"id":44998824,"name":"paasta","full_name":"Yelp/paasta","owner":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Yelp/paasta","description":"An open, distributed platform as a service","fork":false,"url":"https://api.github.com/repos/Yelp/paasta","forks_url":"https://api.github.com/repos/Yelp/paasta/forks","keys_url":"https://api.github.com/repos/Yelp/paasta/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Yelp/paasta/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Yelp/paasta/teams","hooks_url":"https://api.github.com/repos/Yelp/paasta/hooks","issue_events_url":"https://api.github.com/repos/Yelp/paasta/issues/events{/number}","events_url":"https://api.github.com/repos/Yelp/paasta/events","assignees_url":"https://api.github.com/repos/Yelp/paasta/assignees{/user}","branches_url":"https://api.github.com/repos/Yelp/paasta/branches{/branch}","tags_url":"https://api.github.com/repos/Yelp/paasta/tags","blobs_url":"https://api.github.com/repos/Yelp/paasta/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Yelp/paasta/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Yelp/paasta/git/refs{/sha}","trees_url":"https://api.github.com/repos/Yelp/paasta/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Yelp/paasta/statuses/{sha}","languages_url":"https://api.github.com/repos/Yelp/paasta/languages","stargazers_url":"https://api.github.com/repos/Yelp/paasta/stargazers","contributors_url":"https://api.github.com/repos/Yelp/paasta/contributors","subscribers_url":"https://api.github.com/repos/Yelp/paasta/subscribers","subscription_url":"https://api.github.com/repos/Yelp/paasta/subscription","commits_url":"https://api.github.com/repos/Yelp/paasta/commits{/sha}","git_commits_url":"https://api.github.com/repos/Yelp/paasta/git/commits{/sha}","comments_url":"https://api.github.com/repos/Yelp/paasta/comments{/number}","issue_comment_url":"https://api.github.com/repos/Yelp/paasta/issues/comments{/number}","contents_url":"https://api.github.com/repos/Yelp/paasta/contents/{+path}","compare_url":"https://api.github.com/repos/Yelp/paasta/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Yelp/paasta/merges","archive_url":"https://api.github.com/repos/Yelp/paasta/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Yelp/paasta/downloads","issues_url":"https://api.github.com/repos/Yelp/paasta/issues{/number}","pulls_url":"https://api.github.com/repos/Yelp/paasta/pulls{/number}","milestones_url":"https://api.github.com/repos/Yelp/paasta/milestones{/number}","notifications_url":"https://api.github.com/repos/Yelp/paasta/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Yelp/paasta/labels{/name}","releases_url":"https://api.github.com/repos/Yelp/paasta/releases{/id}","deployments_url":"https://api.github.com/repos/Yelp/paasta/deployments","created_at":"2015-10-26T21:35:53Z","updated_at":"2017-10-16T09:25:23Z","pushed_at":"2017-10-17T16:49:16Z","git_url":"git://github.com/Yelp/paasta.git","ssh_url":"git@github.com:Yelp/paasta.git","clone_url":"https://github.com/Yelp/paasta.git","svn_url":"https://github.com/Yelp/paasta","homepage":"","size":54145,"stargazers_count":1132,"watchers_count":1132,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":140,"mirror_url":null,"open_issues_count":101,"forks":140,"open_issues":101,"watchers":1132,"default_branch":"master"}},"base":{"label":"Yelp:master","ref":"master","sha":"f7149a642d93d51565cc99ea07d18b818674d199","user":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"repo":{"id":44998824,"name":"paasta","full_name":"Yelp/paasta","owner":{"login":"Yelp","id":49071,"avatar_url":"https://avatars1.githubusercontent.com/u/49071?v=4","gravatar_id":"","url":"https://api.github.com/users/Yelp","html_url":"https://github.com/Yelp","followers_url":"https://api.github.com/users/Yelp/followers","following_url":"https://api.github.com/users/Yelp/following{/other_user}","gists_url":"https://api.github.com/users/Yelp/gists{/gist_id}","starred_url":"https://api.github.com/users/Yelp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Yelp/subscriptions","organizations_url":"https://api.github.com/users/Yelp/orgs","repos_url":"https://api.github.com/users/Yelp/repos","events_url":"https://api.github.com/users/Yelp/events{/privacy}","received_events_url":"https://api.github.com/users/Yelp/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Yelp/paasta","description":"An open, distributed platform as a service","fork":false,"url":"https://api.github.com/repos/Yelp/paasta","forks_url":"https://api.github.com/repos/Yelp/paasta/forks","keys_url":"https://api.github.com/repos/Yelp/paasta/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Yelp/paasta/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Yelp/paasta/teams","hooks_url":"https://api.github.com/repos/Yelp/paasta/hooks","issue_events_url":"https://api.github.com/repos/Yelp/paasta/issues/events{/number}","events_url":"https://api.github.com/repos/Yelp/paasta/events","assignees_url":"https://api.github.com/repos/Yelp/paasta/assignees{/user}","branches_url":"https://api.github.com/repos/Yelp/paasta/branches{/branch}","tags_url":"https://api.github.com/repos/Yelp/paasta/tags","blobs_url":"https://api.github.com/repos/Yelp/paasta/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Yelp/paasta/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Yelp/paasta/git/refs{/sha}","trees_url":"https://api.github.com/repos/Yelp/paasta/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Yelp/paasta/statuses/{sha}","languages_url":"https://api.github.com/repos/Yelp/paasta/languages","stargazers_url":"https://api.github.com/repos/Yelp/paasta/stargazers","contributors_url":"https://api.github.com/repos/Yelp/paasta/contributors","subscribers_url":"https://api.github.com/repos/Yelp/paasta/subscribers","subscription_url":"https://api.github.com/repos/Yelp/paasta/subscription","commits_url":"https://api.github.com/repos/Yelp/paasta/commits{/sha}","git_commits_url":"https://api.github.com/repos/Yelp/paasta/git/commits{/sha}","comments_url":"https://api.github.com/repos/Yelp/paasta/comments{/number}","issue_comment_url":"https://api.github.com/repos/Yelp/paasta/issues/comments{/number}","contents_url":"https://api.github.com/repos/Yelp/paasta/contents/{+path}","compare_url":"https://api.github.com/repos/Yelp/paasta/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Yelp/paasta/merges","archive_url":"https://api.github.com/repos/Yelp/paasta/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Yelp/paasta/downloads","issues_url":"https://api.github.com/repos/Yelp/paasta/issues{/number}","pulls_url":"https://api.github.com/repos/Yelp/paasta/pulls{/number}","milestones_url":"https://api.github.com/repos/Yelp/paasta/milestones{/number}","notifications_url":"https://api.github.com/repos/Yelp/paasta/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Yelp/paasta/labels{/name}","releases_url":"https://api.github.com/repos/Yelp/paasta/releases{/id}","deployments_url":"https://api.github.com/repos/Yelp/paasta/deployments","created_at":"2015-10-26T21:35:53Z","updated_at":"2017-10-16T09:25:23Z","pushed_at":"2017-10-17T16:49:16Z","git_url":"git://github.com/Yelp/paasta.git","ssh_url":"git@github.com:Yelp/paasta.git","clone_url":"https://github.com/Yelp/paasta.git","svn_url":"https://github.com/Yelp/paasta","homepage":"","size":54145,"stargazers_count":1132,"watchers_count":1132,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":140,"mirror_url":null,"open_issues_count":101,"forks":140,"open_issues":101,"watchers":1132,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/1496"},"html":{"href":"https://github.com/Yelp/paasta/pull/1496"},"issue":{"href":"https://api.github.com/repos/Yelp/paasta/issues/1496"},"comments":{"href":"https://api.github.com/repos/Yelp/paasta/issues/1496/comments"},"review_comments":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/1496/comments"},"review_comment":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Yelp/paasta/pulls/1496/commits"},"statuses":{"href":"https://api.github.com/repos/Yelp/paasta/statuses/9aa8d8d8dd300a49a01bd1694b22c7b09ea05a38"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"Rob-Johnson"}}