{"action":"created","comment":{"url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/comments/126109410","pull_request_review_id":48563028,"id":126109410,"diff_hunk":"@@ -85,81 +75,47 @@\n OUTPUT_TIMESTAMP_FMT = '%Y%m%dT%H%M%SZ'\n \n \n-def main(_):\n-    \"\"\"Run the scanner.\"\"\"\n-    if FLAGS.list_engines is True:\n-        _list_rules_engines()\n-        sys.exit(1)\n-\n-    if not FLAGS.engine_name:\n-        LOGGER.warn('Provide an engine name')\n-        sys.exit(1)\n-    else:\n-        rules_engine_name = FLAGS.engine_name\n-\n-    LOGGER.info('Using rules engine: %s', rules_engine_name)\n-\n-    LOGGER.info('Initializing the rules engine:\\nUsing rules: %s', FLAGS.rules)\n-\n-    if not FLAGS.rules:\n-        LOGGER.warn(('Provide a rules file. '\n-                     'Use \"forseti_scanner --helpfull\" for help.'))\n-        sys.exit(1)\n-\n-    forseti_config = FLAGS.forseti_config\n-    if forseti_config is None:\n-        LOGGER.error('Path to Forseti Security config needs to be specified.')\n-        sys.exit()\n-\n-    try:\n-        configs = file_loader.read_and_parse_file(forseti_config)\n-    except IOError:\n-        LOGGER.error('Unable to open Forseti Security config file. '\n-                     'Please check your path and filename and try again.')\n-        sys.exit()\n-    configs = file_loader.read_and_parse_file(forseti_config)\n-    global_configs = configs.get('global')\n-    scanner_configs = configs.get('scanner')\n-\n-    snapshot_timestamp = _get_timestamp(global_configs)\n-    if not snapshot_timestamp:\n-        LOGGER.warn('No snapshot timestamp found. Exiting.')\n-        sys.exit()\n-\n-    # Load scanner from map\n-    scanner = sm.SCANNER_MAP[rules_engine_name](global_configs,\n-                                                snapshot_timestamp)\n-\n-    # The Groups Scanner uses a different approach to apply and\n-    # evaluate the rules.  Consolidate on next scanner design.\n-    if rules_engine_name == 'GroupsRulesEngine':\n-        all_violations = scanner.run(FLAGS.rules)\n-        resource_counts = None\n-    else:\n-        # Instantiate rules engine with supplied rules file\n-        rules_engine = em.ENGINE_TO_DATA_MAP[rules_engine_name](\n-            rules_file_path=FLAGS.rules, snapshot_timestamp=snapshot_timestamp)\n-        rules_engine.build_rule_book(global_configs)\n-\n-        iter_objects, resource_counts = scanner.run()\n-\n-        # Load violations processing function\n-        all_violations = scanner.find_violations(\n-            itertools.chain(\n-                *iter_objects),\n-            rules_engine)\n-\n-    # If there are violations, send results.\n-    flattening_scheme = sm.FLATTENING_MAP[rules_engine_name]\n-    if all_violations:\n-        _output_results(global_configs,\n-                        scanner_configs,\n-                        all_violations,\n-                        snapshot_timestamp,\n-                        resource_counts=resource_counts,\n-                        flattening_scheme=flattening_scheme)\n-\n-    LOGGER.info('Scan complete!')\n+def _get_runnable_scanners(global_configs, scanner_configs, snapshot_timestamp):\n+        # Load scanner from map\n+\n+    runnable_scanners = []\n+\n+    scanner = iam_rules_scanner.IamPolicyScanner(\n+        global_configs,\n+        scanner_configs,\n+        snapshot_timestamp,\n+        'samples/scanner/rules.yaml')\n+    runnable_scanners.append(scanner)\n+\n+    scanner = groups_scanner.GroupsScanner(\n+        global_configs,\n+        scanner_configs,\n+        snapshot_timestamp,\n+        'samples/scanner/group_rules.yaml')","path":"google/cloud/security/scanner/scanner.py","position":139,"original_position":139,"commit_id":"51f48bc01bb056dc61e3722ca60761f14136d0f8","original_commit_id":"51f48bc01bb056dc61e3722ca60761f14136d0f8","user":{"login":"thenenadx","id":8319858,"avatar_url":"https://avatars3.githubusercontent.com/u/8319858?v=3","gravatar_id":"","url":"https://api.github.com/users/thenenadx","html_url":"https://github.com/thenenadx","followers_url":"https://api.github.com/users/thenenadx/followers","following_url":"https://api.github.com/users/thenenadx/following{/other_user}","gists_url":"https://api.github.com/users/thenenadx/gists{/gist_id}","starred_url":"https://api.github.com/users/thenenadx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thenenadx/subscriptions","organizations_url":"https://api.github.com/users/thenenadx/orgs","repos_url":"https://api.github.com/users/thenenadx/repos","events_url":"https://api.github.com/users/thenenadx/events{/privacy}","received_events_url":"https://api.github.com/users/thenenadx/received_events","type":"User","site_admin":false},"body":"See above","created_at":"2017-07-07T09:46:26Z","updated_at":"2017-07-07T09:46:26Z","html_url":"https://github.com/GoogleCloudPlatform/forseti-security/pull/439#discussion_r126109410","pull_request_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/439","_links":{"self":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/comments/126109410"},"html":{"href":"https://github.com/GoogleCloudPlatform/forseti-security/pull/439#discussion_r126109410"},"pull_request":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/439"}}},"pull_request":{"url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/439","id":129305694,"html_url":"https://github.com/GoogleCloudPlatform/forseti-security/pull/439","diff_url":"https://github.com/GoogleCloudPlatform/forseti-security/pull/439.diff","patch_url":"https://github.com/GoogleCloudPlatform/forseti-security/pull/439.patch","issue_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/439","number":439,"state":"open","locked":false,"title":"Batch Mode for Scanners","user":{"login":"blueandgold","id":6977544,"avatar_url":"https://avatars0.githubusercontent.com/u/6977544?v=3","gravatar_id":"","url":"https://api.github.com/users/blueandgold","html_url":"https://github.com/blueandgold","followers_url":"https://api.github.com/users/blueandgold/followers","following_url":"https://api.github.com/users/blueandgold/following{/other_user}","gists_url":"https://api.github.com/users/blueandgold/gists{/gist_id}","starred_url":"https://api.github.com/users/blueandgold/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/blueandgold/subscriptions","organizations_url":"https://api.github.com/users/blueandgold/orgs","repos_url":"https://api.github.com/users/blueandgold/repos","events_url":"https://api.github.com/users/blueandgold/events{/privacy}","received_events_url":"https://api.github.com/users/blueandgold/received_events","type":"User","site_admin":false},"body":"This PR started out to make the scanner configurable.  Which then is really about making the scanners to be runnable in batch mode.  In order to make this happen, the individual scanners have to be more self-contained and modular.  This then make scanner.py a lot of simpler, and the individual scanners to be more flexible in their actual implementations.","created_at":"2017-07-07T00:03:58Z","updated_at":"2017-07-07T09:46:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"536874de7639f42039920fe44d877e917ce51796","assignee":null,"assignees":[],"requested_reviewers":[{"login":"carise","id":14340,"avatar_url":"https://avatars3.githubusercontent.com/u/14340?v=3","gravatar_id":"","url":"https://api.github.com/users/carise","html_url":"https://github.com/carise","followers_url":"https://api.github.com/users/carise/followers","following_url":"https://api.github.com/users/carise/following{/other_user}","gists_url":"https://api.github.com/users/carise/gists{/gist_id}","starred_url":"https://api.github.com/users/carise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/carise/subscriptions","organizations_url":"https://api.github.com/users/carise/orgs","repos_url":"https://api.github.com/users/carise/repos","events_url":"https://api.github.com/users/carise/events{/privacy}","received_events_url":"https://api.github.com/users/carise/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/439/commits","review_comments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/439/comments","review_comment_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/comments{/number}","comments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/439/comments","statuses_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/statuses/51f48bc01bb056dc61e3722ca60761f14136d0f8","head":{"label":"GoogleCloudPlatform:batchscanner","ref":"batchscanner","sha":"51f48bc01bb056dc61e3722ca60761f14136d0f8","user":{"login":"GoogleCloudPlatform","id":2810941,"avatar_url":"https://avatars3.githubusercontent.com/u/2810941?v=3","gravatar_id":"","url":"https://api.github.com/users/GoogleCloudPlatform","html_url":"https://github.com/GoogleCloudPlatform","followers_url":"https://api.github.com/users/GoogleCloudPlatform/followers","following_url":"https://api.github.com/users/GoogleCloudPlatform/following{/other_user}","gists_url":"https://api.github.com/users/GoogleCloudPlatform/gists{/gist_id}","starred_url":"https://api.github.com/users/GoogleCloudPlatform/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GoogleCloudPlatform/subscriptions","organizations_url":"https://api.github.com/users/GoogleCloudPlatform/orgs","repos_url":"https://api.github.com/users/GoogleCloudPlatform/repos","events_url":"https://api.github.com/users/GoogleCloudPlatform/events{/privacy}","received_events_url":"https://api.github.com/users/GoogleCloudPlatform/received_events","type":"Organization","site_admin":false},"repo":{"id":82210080,"name":"forseti-security","full_name":"GoogleCloudPlatform/forseti-security","owner":{"login":"GoogleCloudPlatform","id":2810941,"avatar_url":"https://avatars3.githubusercontent.com/u/2810941?v=3","gravatar_id":"","url":"https://api.github.com/users/GoogleCloudPlatform","html_url":"https://github.com/GoogleCloudPlatform","followers_url":"https://api.github.com/users/GoogleCloudPlatform/followers","following_url":"https://api.github.com/users/GoogleCloudPlatform/following{/other_user}","gists_url":"https://api.github.com/users/GoogleCloudPlatform/gists{/gist_id}","starred_url":"https://api.github.com/users/GoogleCloudPlatform/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GoogleCloudPlatform/subscriptions","organizations_url":"https://api.github.com/users/GoogleCloudPlatform/orgs","repos_url":"https://api.github.com/users/GoogleCloudPlatform/repos","events_url":"https://api.github.com/users/GoogleCloudPlatform/events{/privacy}","received_events_url":"https://api.github.com/users/GoogleCloudPlatform/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/GoogleCloudPlatform/forseti-security","description":null,"fork":false,"url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security","forks_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/forks","keys_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/keys{/key_id}","collaborators_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/teams","hooks_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/hooks","issue_events_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/events{/number}","events_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/events","assignees_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/assignees{/user}","branches_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/branches{/branch}","tags_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/tags","blobs_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/refs{/sha}","trees_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/trees{/sha}","statuses_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/statuses/{sha}","languages_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/languages","stargazers_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/stargazers","contributors_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/contributors","subscribers_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/subscribers","subscription_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/subscription","commits_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/commits{/sha}","git_commits_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/commits{/sha}","comments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/comments{/number}","issue_comment_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/comments{/number}","contents_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/contents/{+path}","compare_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/compare/{base}...{head}","merges_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/merges","archive_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/downloads","issues_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues{/number}","pulls_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls{/number}","milestones_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/milestones{/number}","notifications_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/labels{/name}","releases_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/releases{/id}","deployments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/deployments","created_at":"2017-02-16T18:01:26Z","updated_at":"2017-07-06T14:43:22Z","pushed_at":"2017-07-07T06:10:04Z","git_url":"git://github.com/GoogleCloudPlatform/forseti-security.git","ssh_url":"git@github.com:GoogleCloudPlatform/forseti-security.git","clone_url":"https://github.com/GoogleCloudPlatform/forseti-security.git","svn_url":"https://github.com/GoogleCloudPlatform/forseti-security","homepage":"http://forsetisecurity.org","size":3947,"stargazers_count":60,"watchers_count":60,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":false,"has_wiki":false,"has_pages":true,"forks_count":26,"mirror_url":null,"open_issues_count":63,"forks":26,"open_issues":63,"watchers":60,"default_branch":"master"}},"base":{"label":"GoogleCloudPlatform:master","ref":"master","sha":"f90968bc4f794961f410d0cdd88c6740e707a57c","user":{"login":"GoogleCloudPlatform","id":2810941,"avatar_url":"https://avatars3.githubusercontent.com/u/2810941?v=3","gravatar_id":"","url":"https://api.github.com/users/GoogleCloudPlatform","html_url":"https://github.com/GoogleCloudPlatform","followers_url":"https://api.github.com/users/GoogleCloudPlatform/followers","following_url":"https://api.github.com/users/GoogleCloudPlatform/following{/other_user}","gists_url":"https://api.github.com/users/GoogleCloudPlatform/gists{/gist_id}","starred_url":"https://api.github.com/users/GoogleCloudPlatform/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GoogleCloudPlatform/subscriptions","organizations_url":"https://api.github.com/users/GoogleCloudPlatform/orgs","repos_url":"https://api.github.com/users/GoogleCloudPlatform/repos","events_url":"https://api.github.com/users/GoogleCloudPlatform/events{/privacy}","received_events_url":"https://api.github.com/users/GoogleCloudPlatform/received_events","type":"Organization","site_admin":false},"repo":{"id":82210080,"name":"forseti-security","full_name":"GoogleCloudPlatform/forseti-security","owner":{"login":"GoogleCloudPlatform","id":2810941,"avatar_url":"https://avatars3.githubusercontent.com/u/2810941?v=3","gravatar_id":"","url":"https://api.github.com/users/GoogleCloudPlatform","html_url":"https://github.com/GoogleCloudPlatform","followers_url":"https://api.github.com/users/GoogleCloudPlatform/followers","following_url":"https://api.github.com/users/GoogleCloudPlatform/following{/other_user}","gists_url":"https://api.github.com/users/GoogleCloudPlatform/gists{/gist_id}","starred_url":"https://api.github.com/users/GoogleCloudPlatform/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GoogleCloudPlatform/subscriptions","organizations_url":"https://api.github.com/users/GoogleCloudPlatform/orgs","repos_url":"https://api.github.com/users/GoogleCloudPlatform/repos","events_url":"https://api.github.com/users/GoogleCloudPlatform/events{/privacy}","received_events_url":"https://api.github.com/users/GoogleCloudPlatform/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/GoogleCloudPlatform/forseti-security","description":null,"fork":false,"url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security","forks_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/forks","keys_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/keys{/key_id}","collaborators_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/teams","hooks_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/hooks","issue_events_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/events{/number}","events_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/events","assignees_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/assignees{/user}","branches_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/branches{/branch}","tags_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/tags","blobs_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/refs{/sha}","trees_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/trees{/sha}","statuses_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/statuses/{sha}","languages_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/languages","stargazers_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/stargazers","contributors_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/contributors","subscribers_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/subscribers","subscription_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/subscription","commits_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/commits{/sha}","git_commits_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/git/commits{/sha}","comments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/comments{/number}","issue_comment_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/comments{/number}","contents_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/contents/{+path}","compare_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/compare/{base}...{head}","merges_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/merges","archive_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/downloads","issues_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues{/number}","pulls_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls{/number}","milestones_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/milestones{/number}","notifications_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/labels{/name}","releases_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/releases{/id}","deployments_url":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/deployments","created_at":"2017-02-16T18:01:26Z","updated_at":"2017-07-06T14:43:22Z","pushed_at":"2017-07-07T06:10:04Z","git_url":"git://github.com/GoogleCloudPlatform/forseti-security.git","ssh_url":"git@github.com:GoogleCloudPlatform/forseti-security.git","clone_url":"https://github.com/GoogleCloudPlatform/forseti-security.git","svn_url":"https://github.com/GoogleCloudPlatform/forseti-security","homepage":"http://forsetisecurity.org","size":3947,"stargazers_count":60,"watchers_count":60,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":false,"has_wiki":false,"has_pages":true,"forks_count":26,"mirror_url":null,"open_issues_count":63,"forks":26,"open_issues":63,"watchers":60,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/439"},"html":{"href":"https://github.com/GoogleCloudPlatform/forseti-security/pull/439"},"issue":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/439"},"comments":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/issues/439/comments"},"review_comments":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/439/comments"},"review_comment":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/pulls/439/commits"},"statuses":{"href":"https://api.github.com/repos/GoogleCloudPlatform/forseti-security/statuses/51f48bc01bb056dc61e3722ca60761f14136d0f8"}}}}
{"actor":{"display_login":"thenenadx"}}