{"action":"created","comment":{"url":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/comments/105533091","pull_request_review_id":26422283,"id":105533091,"diff_hunk":"@@ -0,0 +1,104 @@\n+#!/bin/env python\n+\n+import os\n+import sys\n+import glob\n+import subprocess\n+\n+\n+def main():\n+\n+    failed = False\n+    hosts = []\n+    for host in sys.argv[1:]:\n+        hosts.append(Host(host))\n+\n+    for test in glob.iglob(os.path.join(sys.path[0], \"test-*.sh\")):\n+        host = wait_for_next_available_host(hosts)\n+        rc = host.flush()\n+        failed = failed or rc != 0\n+        host.dispatch(test)\n+\n+    for host in hosts:\n+        rc = host.flush()\n+        failed = failed or rc != 0\n+\n+    # fetch the journal from all the hosts which had a failure\n+    fetcher = os.path.join(sys.path[0], \"fetch-journal.sh\")\n+    for host in hosts:\n+        if host.saw_fail:\n+            fetcher_env = dict(os.environ)\n+            fetcher_env.update({'VM': host.hostname,\n+                                'JOURNAL_LOG':\n+                                \"vmcheck/%s.journal.log\" % host.hostname})\n+            subprocess.check_call([fetcher], env=fetcher_env)\n+\n+    return 1 if failed else 0\n+\n+\n+def wait_for_next_available_host(hosts):\n+    while True:\n+        for host in hosts:\n+            if host.is_done():\n+                return host\n+        os.wait()\n+\n+\n+class Host:\n+\n+    def __init__(self, hostname):\n+        self.hostname = hostname\n+        self.test = \"\"\n+        self._p = None\n+        self.saw_fail = False\n+\n+    def is_done(self):\n+        if not self._p:\n+            return True\n+        return self._p.poll() is not None\n+\n+    def dispatch(self, test):","path":"tests/vmcheck/multitest.py","position":60,"original_position":60,"commit_id":"e6abefd3548908934e5f54b47350ab6cad945807","original_commit_id":"c67a1e491dae27499a534dcb702aa0873a98a3ba","user":{"login":"cgwalters","id":244096,"avatar_url":"https://avatars3.githubusercontent.com/u/244096?v=3","gravatar_id":"","url":"https://api.github.com/users/cgwalters","html_url":"https://github.com/cgwalters","followers_url":"https://api.github.com/users/cgwalters/followers","following_url":"https://api.github.com/users/cgwalters/following{/other_user}","gists_url":"https://api.github.com/users/cgwalters/gists{/gist_id}","starred_url":"https://api.github.com/users/cgwalters/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cgwalters/subscriptions","organizations_url":"https://api.github.com/users/cgwalters/orgs","repos_url":"https://api.github.com/users/cgwalters/repos","events_url":"https://api.github.com/users/cgwalters/events{/privacy}","received_events_url":"https://api.github.com/users/cgwalters/received_events","type":"User","site_admin":false},"body":"I'm thinking this code is something we could convince e.g. GNU `parallel` (or one of [the clones](https://github.com/mmstick/parallel) into doing for us, but...eh.  I only kind of learned the parallel mini-language one time.","created_at":"2017-03-11T13:13:58Z","updated_at":"2017-03-11T13:15:03Z","html_url":"https://github.com/projectatomic/rpm-ostree/pull/675#discussion_r105533091","pull_request_url":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/675","_links":{"self":{"href":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/comments/105533091"},"html":{"href":"https://github.com/projectatomic/rpm-ostree/pull/675#discussion_r105533091"},"pull_request":{"href":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/675"}}},"pull_request":{"url":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/675","id":110240123,"html_url":"https://github.com/projectatomic/rpm-ostree/pull/675","diff_url":"https://github.com/projectatomic/rpm-ostree/pull/675.diff","patch_url":"https://github.com/projectatomic/rpm-ostree/pull/675.patch","issue_url":"https://api.github.com/repos/projectatomic/rpm-ostree/issues/675","number":675,"state":"open","locked":false,"title":"vmcheck: add support for parallel runs","user":{"login":"jlebon","id":11934099,"avatar_url":"https://avatars0.githubusercontent.com/u/11934099?v=3","gravatar_id":"","url":"https://api.github.com/users/jlebon","html_url":"https://github.com/jlebon","followers_url":"https://api.github.com/users/jlebon/followers","following_url":"https://api.github.com/users/jlebon/following{/other_user}","gists_url":"https://api.github.com/users/jlebon/gists{/gist_id}","starred_url":"https://api.github.com/users/jlebon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jlebon/subscriptions","organizations_url":"https://api.github.com/users/jlebon/orgs","repos_url":"https://api.github.com/users/jlebon/repos","events_url":"https://api.github.com/users/jlebon/events{/privacy}","received_events_url":"https://api.github.com/users/jlebon/received_events","type":"User","site_admin":false},"body":"This should help a bit until we move to a more scalable model (discussion in https://github.com/projectatomic/rpm-ostree/issues/662).\r\n\r\nI initially wrote `multitest.py` in bash... boy, what a mistake that was.\r\n\r\nThis should cut down `vmcheck` to about 20-25 mins! (Depending on how the jobs are scheduled in a particular run).","created_at":"2017-03-11T01:37:14Z","updated_at":"2017-03-11T13:15:03Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bd21207e833f710fd188176be002f72fe5944174","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/675/commits","review_comments_url":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/675/comments","review_comment_url":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/comments{/number}","comments_url":"https://api.github.com/repos/projectatomic/rpm-ostree/issues/675/comments","statuses_url":"https://api.github.com/repos/projectatomic/rpm-ostree/statuses/e6abefd3548908934e5f54b47350ab6cad945807","head":{"label":"jlebon:pr/vmcheck-fast","ref":"pr/vmcheck-fast","sha":"e6abefd3548908934e5f54b47350ab6cad945807","user":{"login":"jlebon","id":11934099,"avatar_url":"https://avatars0.githubusercontent.com/u/11934099?v=3","gravatar_id":"","url":"https://api.github.com/users/jlebon","html_url":"https://github.com/jlebon","followers_url":"https://api.github.com/users/jlebon/followers","following_url":"https://api.github.com/users/jlebon/following{/other_user}","gists_url":"https://api.github.com/users/jlebon/gists{/gist_id}","starred_url":"https://api.github.com/users/jlebon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jlebon/subscriptions","organizations_url":"https://api.github.com/users/jlebon/orgs","repos_url":"https://api.github.com/users/jlebon/repos","events_url":"https://api.github.com/users/jlebon/events{/privacy}","received_events_url":"https://api.github.com/users/jlebon/received_events","type":"User","site_admin":false},"repo":{"id":43963435,"name":"rpm-ostree","full_name":"jlebon/rpm-ostree","owner":{"login":"jlebon","id":11934099,"avatar_url":"https://avatars0.githubusercontent.com/u/11934099?v=3","gravatar_id":"","url":"https://api.github.com/users/jlebon","html_url":"https://github.com/jlebon","followers_url":"https://api.github.com/users/jlebon/followers","following_url":"https://api.github.com/users/jlebon/following{/other_user}","gists_url":"https://api.github.com/users/jlebon/gists{/gist_id}","starred_url":"https://api.github.com/users/jlebon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jlebon/subscriptions","organizations_url":"https://api.github.com/users/jlebon/orgs","repos_url":"https://api.github.com/users/jlebon/repos","events_url":"https://api.github.com/users/jlebon/events{/privacy}","received_events_url":"https://api.github.com/users/jlebon/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jlebon/rpm-ostree","description":"Store RPMs in OSTree repository, and atomically upgrade from commits","fork":true,"url":"https://api.github.com/repos/jlebon/rpm-ostree","forks_url":"https://api.github.com/repos/jlebon/rpm-ostree/forks","keys_url":"https://api.github.com/repos/jlebon/rpm-ostree/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jlebon/rpm-ostree/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jlebon/rpm-ostree/teams","hooks_url":"https://api.github.com/repos/jlebon/rpm-ostree/hooks","issue_events_url":"https://api.github.com/repos/jlebon/rpm-ostree/issues/events{/number}","events_url":"https://api.github.com/repos/jlebon/rpm-ostree/events","assignees_url":"https://api.github.com/repos/jlebon/rpm-ostree/assignees{/user}","branches_url":"https://api.github.com/repos/jlebon/rpm-ostree/branches{/branch}","tags_url":"https://api.github.com/repos/jlebon/rpm-ostree/tags","blobs_url":"https://api.github.com/repos/jlebon/rpm-ostree/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jlebon/rpm-ostree/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jlebon/rpm-ostree/git/refs{/sha}","trees_url":"https://api.github.com/repos/jlebon/rpm-ostree/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jlebon/rpm-ostree/statuses/{sha}","languages_url":"https://api.github.com/repos/jlebon/rpm-ostree/languages","stargazers_url":"https://api.github.com/repos/jlebon/rpm-ostree/stargazers","contributors_url":"https://api.github.com/repos/jlebon/rpm-ostree/contributors","subscribers_url":"https://api.github.com/repos/jlebon/rpm-ostree/subscribers","subscription_url":"https://api.github.com/repos/jlebon/rpm-ostree/subscription","commits_url":"https://api.github.com/repos/jlebon/rpm-ostree/commits{/sha}","git_commits_url":"https://api.github.com/repos/jlebon/rpm-ostree/git/commits{/sha}","comments_url":"https://api.github.com/repos/jlebon/rpm-ostree/comments{/number}","issue_comment_url":"https://api.github.com/repos/jlebon/rpm-ostree/issues/comments{/number}","contents_url":"https://api.github.com/repos/jlebon/rpm-ostree/contents/{+path}","compare_url":"https://api.github.com/repos/jlebon/rpm-ostree/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jlebon/rpm-ostree/merges","archive_url":"https://api.github.com/repos/jlebon/rpm-ostree/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jlebon/rpm-ostree/downloads","issues_url":"https://api.github.com/repos/jlebon/rpm-ostree/issues{/number}","pulls_url":"https://api.github.com/repos/jlebon/rpm-ostree/pulls{/number}","milestones_url":"https://api.github.com/repos/jlebon/rpm-ostree/milestones{/number}","notifications_url":"https://api.github.com/repos/jlebon/rpm-ostree/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jlebon/rpm-ostree/labels{/name}","releases_url":"https://api.github.com/repos/jlebon/rpm-ostree/releases{/id}","deployments_url":"https://api.github.com/repos/jlebon/rpm-ostree/deployments","created_at":"2015-10-09T15:15:14Z","updated_at":"2016-02-19T17:40:10Z","pushed_at":"2017-03-11T01:32:11Z","git_url":"git://github.com/jlebon/rpm-ostree.git","ssh_url":"git@github.com:jlebon/rpm-ostree.git","clone_url":"https://github.com/jlebon/rpm-ostree.git","svn_url":"https://github.com/jlebon/rpm-ostree","homepage":"","size":3435,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"projectatomic:master","ref":"master","sha":"058d7e356e8e2ed7d3770a5cf8f9b9e85f140921","user":{"login":"projectatomic","id":6852258,"avatar_url":"https://avatars2.githubusercontent.com/u/6852258?v=3","gravatar_id":"","url":"https://api.github.com/users/projectatomic","html_url":"https://github.com/projectatomic","followers_url":"https://api.github.com/users/projectatomic/followers","following_url":"https://api.github.com/users/projectatomic/following{/other_user}","gists_url":"https://api.github.com/users/projectatomic/gists{/gist_id}","starred_url":"https://api.github.com/users/projectatomic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/projectatomic/subscriptions","organizations_url":"https://api.github.com/users/projectatomic/orgs","repos_url":"https://api.github.com/users/projectatomic/repos","events_url":"https://api.github.com/users/projectatomic/events{/privacy}","received_events_url":"https://api.github.com/users/projectatomic/received_events","type":"Organization","site_admin":false},"repo":{"id":15369584,"name":"rpm-ostree","full_name":"projectatomic/rpm-ostree","owner":{"login":"projectatomic","id":6852258,"avatar_url":"https://avatars2.githubusercontent.com/u/6852258?v=3","gravatar_id":"","url":"https://api.github.com/users/projectatomic","html_url":"https://github.com/projectatomic","followers_url":"https://api.github.com/users/projectatomic/followers","following_url":"https://api.github.com/users/projectatomic/following{/other_user}","gists_url":"https://api.github.com/users/projectatomic/gists{/gist_id}","starred_url":"https://api.github.com/users/projectatomic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/projectatomic/subscriptions","organizations_url":"https://api.github.com/users/projectatomic/orgs","repos_url":"https://api.github.com/users/projectatomic/repos","events_url":"https://api.github.com/users/projectatomic/events{/privacy}","received_events_url":"https://api.github.com/users/projectatomic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/projectatomic/rpm-ostree","description":"Store RPMs in OSTree repository, and atomically upgrade from commits","fork":false,"url":"https://api.github.com/repos/projectatomic/rpm-ostree","forks_url":"https://api.github.com/repos/projectatomic/rpm-ostree/forks","keys_url":"https://api.github.com/repos/projectatomic/rpm-ostree/keys{/key_id}","collaborators_url":"https://api.github.com/repos/projectatomic/rpm-ostree/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/projectatomic/rpm-ostree/teams","hooks_url":"https://api.github.com/repos/projectatomic/rpm-ostree/hooks","issue_events_url":"https://api.github.com/repos/projectatomic/rpm-ostree/issues/events{/number}","events_url":"https://api.github.com/repos/projectatomic/rpm-ostree/events","assignees_url":"https://api.github.com/repos/projectatomic/rpm-ostree/assignees{/user}","branches_url":"https://api.github.com/repos/projectatomic/rpm-ostree/branches{/branch}","tags_url":"https://api.github.com/repos/projectatomic/rpm-ostree/tags","blobs_url":"https://api.github.com/repos/projectatomic/rpm-ostree/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/projectatomic/rpm-ostree/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/projectatomic/rpm-ostree/git/refs{/sha}","trees_url":"https://api.github.com/repos/projectatomic/rpm-ostree/git/trees{/sha}","statuses_url":"https://api.github.com/repos/projectatomic/rpm-ostree/statuses/{sha}","languages_url":"https://api.github.com/repos/projectatomic/rpm-ostree/languages","stargazers_url":"https://api.github.com/repos/projectatomic/rpm-ostree/stargazers","contributors_url":"https://api.github.com/repos/projectatomic/rpm-ostree/contributors","subscribers_url":"https://api.github.com/repos/projectatomic/rpm-ostree/subscribers","subscription_url":"https://api.github.com/repos/projectatomic/rpm-ostree/subscription","commits_url":"https://api.github.com/repos/projectatomic/rpm-ostree/commits{/sha}","git_commits_url":"https://api.github.com/repos/projectatomic/rpm-ostree/git/commits{/sha}","comments_url":"https://api.github.com/repos/projectatomic/rpm-ostree/comments{/number}","issue_comment_url":"https://api.github.com/repos/projectatomic/rpm-ostree/issues/comments{/number}","contents_url":"https://api.github.com/repos/projectatomic/rpm-ostree/contents/{+path}","compare_url":"https://api.github.com/repos/projectatomic/rpm-ostree/compare/{base}...{head}","merges_url":"https://api.github.com/repos/projectatomic/rpm-ostree/merges","archive_url":"https://api.github.com/repos/projectatomic/rpm-ostree/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/projectatomic/rpm-ostree/downloads","issues_url":"https://api.github.com/repos/projectatomic/rpm-ostree/issues{/number}","pulls_url":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls{/number}","milestones_url":"https://api.github.com/repos/projectatomic/rpm-ostree/milestones{/number}","notifications_url":"https://api.github.com/repos/projectatomic/rpm-ostree/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/projectatomic/rpm-ostree/labels{/name}","releases_url":"https://api.github.com/repos/projectatomic/rpm-ostree/releases{/id}","deployments_url":"https://api.github.com/repos/projectatomic/rpm-ostree/deployments","created_at":"2013-12-22T03:16:38Z","updated_at":"2017-03-07T19:14:27Z","pushed_at":"2017-03-11T01:37:15Z","git_url":"git://github.com/projectatomic/rpm-ostree.git","ssh_url":"git@github.com:projectatomic/rpm-ostree.git","clone_url":"https://github.com/projectatomic/rpm-ostree.git","svn_url":"https://github.com/projectatomic/rpm-ostree","homepage":"","size":3431,"stargazers_count":143,"watchers_count":143,"language":"C","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":61,"mirror_url":null,"open_issues_count":107,"forks":61,"open_issues":107,"watchers":143,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/675"},"html":{"href":"https://github.com/projectatomic/rpm-ostree/pull/675"},"issue":{"href":"https://api.github.com/repos/projectatomic/rpm-ostree/issues/675"},"comments":{"href":"https://api.github.com/repos/projectatomic/rpm-ostree/issues/675/comments"},"review_comments":{"href":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/675/comments"},"review_comment":{"href":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/projectatomic/rpm-ostree/pulls/675/commits"},"statuses":{"href":"https://api.github.com/repos/projectatomic/rpm-ostree/statuses/e6abefd3548908934e5f54b47350ab6cad945807"}}}}
{"actor":{"display_login":"cgwalters"}}