{"action":"closed","number":243,"pull_request":{"url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/243","id":117703634,"html_url":"https://github.com/GoogleCloudPlatform/forseti-security/pull/243","diff_url":"https://github.com/GoogleCloudPlatform/forseti-security/pull/243.diff","patch_url":"https://github.com/GoogleCloudPlatform/forseti-security/pull/243.patch","issue_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/243","number":243,"state":"closed","locked":false,"title":"Batch insert group members","user":{"login":"gbrindisi","id":287126,"avatar_url":"https://avatars3.githubusercontent.com/u/287126?v=3","gravatar_id":"","url":"https://api.github.com/users/gbrindisi","html_url":"https://github.com/gbrindisi","followers_url":"https://api.github.com/users/gbrindisi/followers","following_url":"https://api.github.com/users/gbrindisi/following{/other_user}","gists_url":"https://api.github.com/users/gbrindisi/gists{/gist_id}","starred_url":"https://api.github.com/users/gbrindisi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gbrindisi/subscriptions","organizations_url":"https://api.github.com/users/gbrindisi/orgs","repos_url":"https://api.github.com/users/gbrindisi/repos","events_url":"https://api.github.com/users/gbrindisi/events{/privacy}","received_events_url":"https://api.github.com/users/gbrindisi/received_events","type":"User","site_admin":false},"body":"Hi there,\r\nin an effort to make the `group_members` pipeline more fast/resilient I've implemented a batch retrieve/commit mechanism. \r\n(This is slightly related to https://github.com/GoogleCloudPlatform/forseti-security/issues/241.)\r\n\r\nIt is very useful in our environment given our number of groups.\r\n\r\nThe modification is two fold:\r\n* in the pipeline, group_ids are chunked and processed in batch: this means both querying and storing results.\r\n* in the DAO, the `load_data` method now supports loading data into an already existing table. This part is not great, I pretty much did a quick and dirty exception based control flow. Can you suggest improvements? \r\n\r\n","created_at":"2017-04-26T12:37:53Z","updated_at":"2017-04-28T17:59:47Z","closed_at":"2017-04-28T17:59:47Z","merged_at":"2017-04-28T17:59:47Z","merge_commit_sha":"b98ccba71d097b2d0572a4a971628c8e435af362","assignee":{"login":"gbrindisi","id":287126,"avatar_url":"https://avatars3.githubusercontent.com/u/287126?v=3","gravatar_id":"","url":"https://api.github.com/users/gbrindisi","html_url":"https://github.com/gbrindisi","followers_url":"https://api.github.com/users/gbrindisi/followers","following_url":"https://api.github.com/users/gbrindisi/following{/other_user}","gists_url":"https://api.github.com/users/gbrindisi/gists{/gist_id}","starred_url":"https://api.github.com/users/gbrindisi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gbrindisi/subscriptions","organizations_url":"https://api.github.com/users/gbrindisi/orgs","repos_url":"https://api.github.com/users/gbrindisi/repos","events_url":"https://api.github.com/users/gbrindisi/events{/privacy}","received_events_url":"https://api.github.com/users/gbrindisi/received_events","type":"User","site_admin":false},"assignees":[{"login":"gbrindisi","id":287126,"avatar_url":"https://avatars3.githubusercontent.com/u/287126?v=3","gravatar_id":"","url":"https://api.github.com/users/gbrindisi","html_url":"https://github.com/gbrindisi","followers_url":"https://api.github.com/users/gbrindisi/followers","following_url":"https://api.github.com/users/gbrindisi/following{/other_user}","gists_url":"https://api.github.com/users/gbrindisi/gists{/gist_id}","starred_url":"https://api.github.com/users/gbrindisi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gbrindisi/subscriptions","organizations_url":"https://api.github.com/users/gbrindisi/orgs","repos_url":"https://api.github.com/users/gbrindisi/repos","events_url":"https://api.github.com/users/gbrindisi/events{/privacy}","received_events_url":"https://api.github.com/users/gbrindisi/received_events","type":"User","site_admin":false}],"milestone":{"url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/milestones/1","html_url":"https://github.com/GoogleCloudPlatform/forseti-security/milestone/1","labels_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/milestones/1/labels","id":2366429,"number":1,"title":"M1","description":"","creator":{"login":"mirons-google","id":22750334,"avatar_url":"https://avatars0.githubusercontent.com/u/22750334?v=3","gravatar_id":"","url":"https://api.github.com/users/mirons-google","html_url":"https://github.com/mirons-google","followers_url":"https://api.github.com/users/mirons-google/followers","following_url":"https://api.github.com/users/mirons-google/following{/other_user}","gists_url":"https://api.github.com/users/mirons-google/gists{/gist_id}","starred_url":"https://api.github.com/users/mirons-google/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mirons-google/subscriptions","organizations_url":"https://api.github.com/users/mirons-google/orgs","repos_url":"https://api.github.com/users/mirons-google/repos","events_url":"https://api.github.com/users/mirons-google/events{/privacy}","received_events_url":"https://api.github.com/users/mirons-google/received_events","type":"User","site_admin":false},"open_issues":15,"closed_issues":33,"state":"open","created_at":"2017-03-06T15:48:06Z","updated_at":"2017-04-28T17:59:47Z","due_on":"2017-06-30T07:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/243/commits","review_comments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/243/comments","review_comment_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/comments{/number}","comments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/243/comments","statuses_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/statuses/176a415f06a719e0c61052f8c83f04dbb006afba","head":{"label":"gbrindisi:batch-insert-group-members","ref":"batch-insert-group-members","sha":"176a415f06a719e0c61052f8c83f04dbb006afba","user":{"login":"gbrindisi","id":287126,"avatar_url":"https://avatars3.githubusercontent.com/u/287126?v=3","gravatar_id":"","url":"https://api.github.com/users/gbrindisi","html_url":"https://github.com/gbrindisi","followers_url":"https://api.github.com/users/gbrindisi/followers","following_url":"https://api.github.com/users/gbrindisi/following{/other_user}","gists_url":"https://api.github.com/users/gbrindisi/gists{/gist_id}","starred_url":"https://api.github.com/users/gbrindisi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gbrindisi/subscriptions","organizations_url":"https://api.github.com/users/gbrindisi/orgs","repos_url":"https://api.github.com/users/gbrindisi/repos","events_url":"https://api.github.com/users/gbrindisi/events{/privacy}","received_events_url":"https://api.github.com/users/gbrindisi/received_events","type":"User","site_admin":false},"repo":{"id":89378377,"name":"forseti-security","full_name":"gbrindisi/forseti-security","owner":{"login":"gbrindisi","id":287126,"avatar_url":"https://avatars3.githubusercontent.com/u/287126?v=3","gravatar_id":"","url":"https://api.github.com/users/gbrindisi","html_url":"https://github.com/gbrindisi","followers_url":"https://api.github.com/users/gbrindisi/followers","following_url":"https://api.github.com/users/gbrindisi/following{/other_user}","gists_url":"https://api.github.com/users/gbrindisi/gists{/gist_id}","starred_url":"https://api.github.com/users/gbrindisi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gbrindisi/subscriptions","organizations_url":"https://api.github.com/users/gbrindisi/orgs","repos_url":"https://api.github.com/users/gbrindisi/repos","events_url":"https://api.github.com/users/gbrindisi/events{/privacy}","received_events_url":"https://api.github.com/users/gbrindisi/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/gbrindisi/forseti-security","description":"Security tools for visibility, detection, and policy enforcement on Google Cloud Platform","fork":true,"url":"https://api.github.com/repos/gbrindisi/forseti-security","forks_url":"https://api.github.com/repos/gbrindisi/forseti-security/forks","keys_url":"https://api.github.com/repos/gbrindisi/forseti-security/keys{/key_id}","collaborators_url":"https://api.github.com/repos/gbrindisi/forseti-security/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/gbrindisi/forseti-security/teams","hooks_url":"https://api.github.com/repos/gbrindisi/forseti-security/hooks","issue_events_url":"https://api.github.com/repos/gbrindisi/forseti-security/issues/events{/number}","events_url":"https://api.github.com/repos/gbrindisi/forseti-security/events","assignees_url":"https://api.github.com/repos/gbrindisi/forseti-security/assignees{/user}","branches_url":"https://api.github.com/repos/gbrindisi/forseti-security/branches{/branch}","tags_url":"https://api.github.com/repos/gbrindisi/forseti-security/tags","blobs_url":"https://api.github.com/repos/gbrindisi/forseti-security/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/gbrindisi/forseti-security/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/gbrindisi/forseti-security/git/refs{/sha}","trees_url":"https://api.github.com/repos/gbrindisi/forseti-security/git/trees{/sha}","statuses_url":"https://api.github.com/repos/gbrindisi/forseti-security/statuses/{sha}","languages_url":"https://api.github.com/repos/gbrindisi/forseti-security/languages","stargazers_url":"https://api.github.com/repos/gbrindisi/forseti-security/stargazers","contributors_url":"https://api.github.com/repos/gbrindisi/forseti-security/contributors","subscribers_url":"https://api.github.com/repos/gbrindisi/forseti-security/subscribers","subscription_url":"https://api.github.com/repos/gbrindisi/forseti-security/subscription","commits_url":"https://api.github.com/repos/gbrindisi/forseti-security/commits{/sha}","git_commits_url":"https://api.github.com/repos/gbrindisi/forseti-security/git/commits{/sha}","comments_url":"https://api.github.com/repos/gbrindisi/forseti-security/comments{/number}","issue_comment_url":"https://api.github.com/repos/gbrindisi/forseti-security/issues/comments{/number}","contents_url":"https://api.github.com/repos/gbrindisi/forseti-security/contents/{+path}","compare_url":"https://api.github.com/repos/gbrindisi/forseti-security/compare/{base}...{head}","merges_url":"https://api.github.com/repos/gbrindisi/forseti-security/merges","archive_url":"https://api.github.com/repos/gbrindisi/forseti-security/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/gbrindisi/forseti-security/downloads","issues_url":"https://api.github.com/repos/gbrindisi/forseti-security/issues{/number}","pulls_url":"https://api.github.com/repos/gbrindisi/forseti-security/pulls{/number}","milestones_url":"https://api.github.com/repos/gbrindisi/forseti-security/milestones{/number}","notifications_url":"https://api.github.com/repos/gbrindisi/forseti-security/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/gbrindisi/forseti-security/labels{/name}","releases_url":"https://api.github.com/repos/gbrindisi/forseti-security/releases{/id}","deployments_url":"https://api.github.com/repos/gbrindisi/forseti-security/deployments","created_at":"2017-04-25T15:43:22Z","updated_at":"2017-04-25T15:43:24Z","pushed_at":"2017-04-28T17:47:12Z","git_url":"git://github.com/gbrindisi/forseti-security.git","ssh_url":"git@github.com:gbrindisi/forseti-security.git","clone_url":"https://github.com/gbrindisi/forseti-security.git","svn_url":"https://github.com/gbrindisi/forseti-security","homepage":"","size":579,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":false,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"GoogleCloudPlatform:master","ref":"master","sha":"2c25af12b319f32b5a71724366bb67cb77b09777","user":{"login":"GoogleCloudPlatform","id":2810941,"avatar_url":"https://avatars3.githubusercontent.com/u/2810941?v=3","gravatar_id":"","url":"https://api.github.com/users/GoogleCloudPlatform","html_url":"https://github.com/GoogleCloudPlatform","followers_url":"https://api.github.com/users/GoogleCloudPlatform/followers","following_url":"https://api.github.com/users/GoogleCloudPlatform/following{/other_user}","gists_url":"https://api.github.com/users/GoogleCloudPlatform/gists{/gist_id}","starred_url":"https://api.github.com/users/GoogleCloudPlatform/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GoogleCloudPlatform/subscriptions","organizations_url":"https://api.github.com/users/GoogleCloudPlatform/orgs","repos_url":"https://api.github.com/users/GoogleCloudPlatform/repos","events_url":"https://api.github.com/users/GoogleCloudPlatform/events{/privacy}","received_events_url":"https://api.github.com/users/GoogleCloudPlatform/received_events","type":"Organization","site_admin":false},"repo":{"id":82210080,"name":"forseti-security","full_name":"GoogleCloudPlatform/forseti-security","owner":{"login":"GoogleCloudPlatform","id":2810941,"avatar_url":"https://avatars3.githubusercontent.com/u/2810941?v=3","gravatar_id":"","url":"https://api.github.com/users/GoogleCloudPlatform","html_url":"https://github.com/GoogleCloudPlatform","followers_url":"https://api.github.com/users/GoogleCloudPlatform/followers","following_url":"https://api.github.com/users/GoogleCloudPlatform/following{/other_user}","gists_url":"https://api.github.com/users/GoogleCloudPlatform/gists{/gist_id}","starred_url":"https://api.github.com/users/GoogleCloudPlatform/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GoogleCloudPlatform/subscriptions","organizations_url":"https://api.github.com/users/GoogleCloudPlatform/orgs","repos_url":"https://api.github.com/users/GoogleCloudPlatform/repos","events_url":"https://api.github.com/users/GoogleCloudPlatform/events{/privacy}","received_events_url":"https://api.github.com/users/GoogleCloudPlatform/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/GoogleCloudPlatform/forseti-security","description":"Security tools for visibility, detection, and policy enforcement on Google Cloud Platform","fork":false,"url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security","forks_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/forks","keys_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/keys{/key_id}","collaborators_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/teams","hooks_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/hooks","issue_events_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/events{/number}","events_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/events","assignees_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/assignees{/user}","branches_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/branches{/branch}","tags_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/tags","blobs_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/refs{/sha}","trees_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/trees{/sha}","statuses_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/statuses/{sha}","languages_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/languages","stargazers_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/stargazers","contributors_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/contributors","subscribers_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/subscribers","subscription_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/subscription","commits_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/commits{/sha}","git_commits_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/commits{/sha}","comments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/comments{/number}","issue_comment_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/comments{/number}","contents_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/contents/{+path}","compare_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/compare/{base}...{head}","merges_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/merges","archive_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/downloads","issues_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues{/number}","pulls_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls{/number}","milestones_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/milestones{/number}","notifications_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/labels{/name}","releases_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/releases{/id}","deployments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/deployments","created_at":"2017-02-16T18:01:26Z","updated_at":"2017-04-25T21:34:53Z","pushed_at":"2017-04-28T17:59:47Z","git_url":"git://github.com/GoogleCloudPlatform/forseti-security.git","ssh_url":"git@github.com:GoogleCloudPlatform/forseti-security.git","clone_url":"https://github.com/GoogleCloudPlatform/forseti-security.git","svn_url":"https://github.com/GoogleCloudPlatform/forseti-security","homepage":"","size":531,"stargazers_count":40,"watchers_count":40,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":false,"has_wiki":false,"has_pages":false,"forks_count":13,"mirror_url":null,"open_issues_count":53,"forks":13,"open_issues":53,"watchers":40,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/243"},"html":{"href":"https://github.com/GoogleCloudPlatform/forseti-security/pull/243"},"issue":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/243"},"comments":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/243/comments"},"review_comments":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/243/comments"},"review_comment":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/243/commits"},"statuses":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/statuses/176a415f06a719e0c61052f8c83f04dbb006afba"}},"merged":true,"mergeable":null,"rebaseable":null,"mergeable_state":"unknown","merged_by":{"login":"blueandgold","id":6977544,"avatar_url":"https://avatars0.githubusercontent.com/u/6977544?v=3","gravatar_id":"","url":"https://api.github.com/users/blueandgold","html_url":"https://github.com/blueandgold","followers_url":"https://api.github.com/users/blueandgold/followers","following_url":"https://api.github.com/users/blueandgold/following{/other_user}","gists_url":"https://api.github.com/users/blueandgold/gists{/gist_id}","starred_url":"https://api.github.com/users/blueandgold/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/blueandgold/subscriptions","organizations_url":"https://api.github.com/users/blueandgold/orgs","repos_url":"https://api.github.com/users/blueandgold/repos","events_url":"https://api.github.com/users/blueandgold/events{/privacy}","received_events_url":"https://api.github.com/users/blueandgold/received_events","type":"User","site_admin":false},"comments":16,"review_comments":18,"maintainer_can_modify":false,"commits":25,"additions":75,"deletions":16,"changed_files":3}}
{"actor":{"display_login":"blueandgold"}}