{"action":"created","comment":{"url":"https://api.github.com/repos/TheTorProject/proteus/pulls/comments/155046068","pull_request_review_id":81300491,"id":155046068,"diff_hunk":"@@ -351,6 +403,184 @@ var ErrAccessDenied = errors.New(\"access denied\")\n // ErrInconsistentState when you try to accept an already accepted task\n var ErrInconsistentState = errors.New(\"task already accepted\")\n \n+// GetCollectors() returns a map of collectors keyed by their type\n+func GetCollectors(db *sqlx.DB) (map[string]interface{}, error) {\n+\tvar (\n+\t\terr error\n+\t)\n+\tcollectors := make(map[string]interface{})\n+\tquery := fmt.Sprintf(`SELECT\n+\t\ttype,\n+\t\taddress,\n+\t\tfront_domain\n+\t\tFROM %s`,\n+\t\tpq.QuoteIdentifier(viper.GetString(\"database.collectors-table\")))\n+\trows, err := db.Query(query)\n+\tif err != nil {\n+\t\tif err == sql.ErrNoRows {\n+\t\t\treturn collectors, nil\n+\t\t}\n+\t\tctx.WithError(err).Error(\"failed to get collectors\")\n+\t\treturn collectors, err\n+\t}\n+\tdefer rows.Close()\n+\tvar (\n+\t\tonion_cos   []string\n+\t\thttps_cos   []string\n+\t\tfronted_cos []map[string]string\n+\t)\n+\tfor rows.Next() {\n+\t\tvar (\n+\t\t\tctype    string\n+\t\t\tcaddress string\n+\t\t\tcfront   sql.NullString\n+\t\t)\n+\t\terr = rows.Scan(&ctype, &caddress, &cfront)\n+\t\tif err != nil {\n+\t\t\tctx.WithError(err).Error(\"failed to get collector row\")\n+\t\t\tcontinue\n+\t\t}\n+\t\tswitch ctype {\n+\t\tcase \"onion\":\n+\t\t\tonion_cos = append(onion_cos, caddress)\n+\t\tcase \"https\":\n+\t\t\thttps_cos = append(https_cos, caddress)\n+\t\tcase \"domain_fronted\":\n+\t\t\tif !cfront.Valid {\n+\t\t\t\tctx.Error(\"domain_fronted collector with bad front domain\")\n+\t\t\t\tcontinue\n+\t\t\t}\n+\t\t\tfronted_co := map[string]string{}\n+\t\t\tfronted_co[\"domain\"] = caddress\n+\t\t\tfronted_co[\"front\"] = cfront.String\n+\t\t\tfronted_cos = append(fronted_cos, fronted_co)\n+\t\tdefault:\n+\t\t\tctx.Error(\"collector with bad type in DB\")\n+\t\t\tcontinue\n+\t\t}\n+\t}\n+\tcollectors[\"onion\"] = onion_cos\n+\tcollectors[\"https\"] = https_cos\n+\tcollectors[\"domain_fronted\"] = fronted_cos\n+\treturn collectors, nil\n+}\n+\n+// GetTestHelpers() returns a map of test helpers keyed by the test name\n+func GetTestHelpers(db *sqlx.DB) (map[string][]string, error) {\n+\tvar (\n+\t\terr error\n+\t)\n+\thelpers := make(map[string][]string)\n+\tquery := fmt.Sprintf(`SELECT\n+\t\ttest_name,\n+\t\taddress\n+\t\tFROM %s`,\n+\t\tpq.QuoteIdentifier(viper.GetString(\"database.test-helpers-table\")))\n+\trows, err := db.Query(query)\n+\tif err != nil {\n+\t\tif err == sql.ErrNoRows {\n+\t\t\treturn helpers, nil\n+\t\t}\n+\t\tctx.WithError(err).Error(\"failed to get test helpers\")\n+\t\treturn helpers, err\n+\t}\n+\tdefer rows.Close()\n+\tfor rows.Next() {\n+\t\tvar (\n+\t\t\ttest_name string\n+\t\t\taddress   string\n+\t\t)\n+\t\terr = rows.Scan(&test_name, &address)\n+\t\tif err != nil {\n+\t\t\tctx.WithError(err).Error(\"failed to get test_helper row\")\n+\t\t\tcontinue\n+\t\t}\n+\t\thelpers[test_name] = append(helpers[test_name], address)\n+\t}\n+\treturn helpers, nil\n+}\n+\n+func BuildTestInputQuery(countries []string, cat_codes []string) (string, error) {\n+\tquery := fmt.Sprintf(`SELECT\n+\t\turl,\n+\t\tcat_code,\n+\t\talpha_2\n+\t\tFROM %s urls\n+\t\tINNER JOIN %s cos ON urls.country_no = cos.country_no\n+\t\tINNER JOIN %s url_cats ON urls.cat_no = url_cats.cat_no`,\n+\t\tpq.QuoteIdentifier(viper.GetString(\"database.urls-table\")),\n+\t\tpq.QuoteIdentifier(viper.GetString(\"database.countries-table\")),\n+\t\tpq.QuoteIdentifier(viper.GetString(\"database.url-categories-table\")))\n+\tif len(countries) > 0 {\n+\t\tquery += fmt.Sprintf(`\n+\t\t\tWHERE (cos.alpha_2 = $1`)\n+\t\tfor i, _ := range countries[1:] {\n+\t\t\tquery += fmt.Sprintf(` OR cos.alpha_2 = $%d`, i+2)\n+\t\t}\n+\t\tquery += `)`\n+\t}\n+\tif len(cat_codes) > 0 {\n+\t\tif len(countries) > 0 {\n+\t\t\tquery += `\n+\t\t\t\tAND `\n+\t\t} else {\n+\t\t\tquery += `\n+\t\t\t\tWHERE `\n+\t\t}\n+\t\tquery += fmt.Sprintf(`(url_cats.cat_code = $%d`, len(countries)+1)","path":"proteus-events/events/events.go","position":null,"original_position":194,"commit_id":"b1a47f556d941e62f5e8f083a3612123ed90d99e","original_commit_id":"33a59a288cca66754589e46055e6fb1fd8e2ccfe","user":{"login":"joelanders","id":1390211,"avatar_url":"https://avatars3.githubusercontent.com/u/1390211?v=4","gravatar_id":"","url":"https://api.github.com/users/joelanders","html_url":"https://github.com/joelanders","followers_url":"https://api.github.com/users/joelanders/followers","following_url":"https://api.github.com/users/joelanders/following{/other_user}","gists_url":"https://api.github.com/users/joelanders/gists{/gist_id}","starred_url":"https://api.github.com/users/joelanders/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joelanders/subscriptions","organizations_url":"https://api.github.com/users/joelanders/orgs","repos_url":"https://api.github.com/users/joelanders/repos","events_url":"https://api.github.com/users/joelanders/events{/privacy}","received_events_url":"https://api.github.com/users/joelanders/received_events","type":"User","site_admin":false},"body":"done","created_at":"2017-12-05T19:07:42Z","updated_at":"2017-12-05T19:07:55Z","html_url":"https://github.com/TheTorProject/proteus/pull/29#discussion_r155046068","pull_request_url":"https://api.github.com/repos/TheTorProject/proteus/pulls/29","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/TheTorProject/proteus/pulls/comments/155046068"},"html":{"href":"https://github.com/TheTorProject/proteus/pull/29#discussion_r155046068"},"pull_request":{"href":"https://api.github.com/repos/TheTorProject/proteus/pulls/29"}},"in_reply_to_id":154896533},"pull_request":{"url":"https://api.github.com/repos/TheTorProject/proteus/pulls/29","id":149803598,"html_url":"https://github.com/TheTorProject/proteus/pull/29","diff_url":"https://github.com/TheTorProject/proteus/pull/29.diff","patch_url":"https://github.com/TheTorProject/proteus/pull/29.patch","issue_url":"https://api.github.com/repos/TheTorProject/proteus/issues/29","number":29,"state":"open","locked":false,"title":"Rendezvous endpoint","user":{"login":"joelanders","id":1390211,"avatar_url":"https://avatars3.githubusercontent.com/u/1390211?v=4","gravatar_id":"","url":"https://api.github.com/users/joelanders","html_url":"https://github.com/joelanders","followers_url":"https://api.github.com/users/joelanders/followers","following_url":"https://api.github.com/users/joelanders/following{/other_user}","gists_url":"https://api.github.com/users/joelanders/gists{/gist_id}","starred_url":"https://api.github.com/users/joelanders/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joelanders/subscriptions","organizations_url":"https://api.github.com/users/joelanders/orgs","repos_url":"https://api.github.com/users/joelanders/repos","events_url":"https://api.github.com/users/joelanders/events{/privacy}","received_events_url":"https://api.github.com/users/joelanders/received_events","type":"User","site_admin":false},"body":"Getting close to the end I think.\r\n\r\n* probe country code comes from looking it up against the probeID in the DB? or is it like a query param?\r\n\r\n* category code can be blank, I assume?\r\n\r\n* should we just return a random set of matching URLs for now? (later we can keep track of what we've sent out and stuff)\r\n\r\n* LIMIT 100 or what?","created_at":"2017-10-31T14:35:30Z","updated_at":"2017-12-05T19:07:55Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b1d0de799de50dbe400c6ba505d254350283602b","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/TheTorProject/proteus/pulls/29/commits","review_comments_url":"https://api.github.com/repos/TheTorProject/proteus/pulls/29/comments","review_comment_url":"https://api.github.com/repos/TheTorProject/proteus/pulls/comments{/number}","comments_url":"https://api.github.com/repos/TheTorProject/proteus/issues/29/comments","statuses_url":"https://api.github.com/repos/TheTorProject/proteus/statuses/b1a47f556d941e62f5e8f083a3612123ed90d99e","head":{"label":"TheTorProject:rendezvous","ref":"rendezvous","sha":"b1a47f556d941e62f5e8f083a3612123ed90d99e","user":{"login":"TheTorProject","id":3301912,"avatar_url":"https://avatars1.githubusercontent.com/u/3301912?v=4","gravatar_id":"","url":"https://api.github.com/users/TheTorProject","html_url":"https://github.com/TheTorProject","followers_url":"https://api.github.com/users/TheTorProject/followers","following_url":"https://api.github.com/users/TheTorProject/following{/other_user}","gists_url":"https://api.github.com/users/TheTorProject/gists{/gist_id}","starred_url":"https://api.github.com/users/TheTorProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TheTorProject/subscriptions","organizations_url":"https://api.github.com/users/TheTorProject/orgs","repos_url":"https://api.github.com/users/TheTorProject/repos","events_url":"https://api.github.com/users/TheTorProject/events{/privacy}","received_events_url":"https://api.github.com/users/TheTorProject/received_events","type":"Organization","site_admin":false},"repo":{"id":83037093,"name":"proteus","full_name":"TheTorProject/proteus","owner":{"login":"TheTorProject","id":3301912,"avatar_url":"https://avatars1.githubusercontent.com/u/3301912?v=4","gravatar_id":"","url":"https://api.github.com/users/TheTorProject","html_url":"https://github.com/TheTorProject","followers_url":"https://api.github.com/users/TheTorProject/followers","following_url":"https://api.github.com/users/TheTorProject/following{/other_user}","gists_url":"https://api.github.com/users/TheTorProject/gists{/gist_id}","starred_url":"https://api.github.com/users/TheTorProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TheTorProject/subscriptions","organizations_url":"https://api.github.com/users/TheTorProject/orgs","repos_url":"https://api.github.com/users/TheTorProject/repos","events_url":"https://api.github.com/users/TheTorProject/events{/privacy}","received_events_url":"https://api.github.com/users/TheTorProject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/TheTorProject/proteus","description":"The OONI Probe Orchestration System","fork":false,"url":"https://api.github.com/repos/TheTorProject/proteus","forks_url":"https://api.github.com/repos/TheTorProject/proteus/forks","keys_url":"https://api.github.com/repos/TheTorProject/proteus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/TheTorProject/proteus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/TheTorProject/proteus/teams","hooks_url":"https://api.github.com/repos/TheTorProject/proteus/hooks","issue_events_url":"https://api.github.com/repos/TheTorProject/proteus/issues/events{/number}","events_url":"https://api.github.com/repos/TheTorProject/proteus/events","assignees_url":"https://api.github.com/repos/TheTorProject/proteus/assignees{/user}","branches_url":"https://api.github.com/repos/TheTorProject/proteus/branches{/branch}","tags_url":"https://api.github.com/repos/TheTorProject/proteus/tags","blobs_url":"https://api.github.com/repos/TheTorProject/proteus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/TheTorProject/proteus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/TheTorProject/proteus/git/refs{/sha}","trees_url":"https://api.github.com/repos/TheTorProject/proteus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/TheTorProject/proteus/statuses/{sha}","languages_url":"https://api.github.com/repos/TheTorProject/proteus/languages","stargazers_url":"https://api.github.com/repos/TheTorProject/proteus/stargazers","contributors_url":"https://api.github.com/repos/TheTorProject/proteus/contributors","subscribers_url":"https://api.github.com/repos/TheTorProject/proteus/subscribers","subscription_url":"https://api.github.com/repos/TheTorProject/proteus/subscription","commits_url":"https://api.github.com/repos/TheTorProject/proteus/commits{/sha}","git_commits_url":"https://api.github.com/repos/TheTorProject/proteus/git/commits{/sha}","comments_url":"https://api.github.com/repos/TheTorProject/proteus/comments{/number}","issue_comment_url":"https://api.github.com/repos/TheTorProject/proteus/issues/comments{/number}","contents_url":"https://api.github.com/repos/TheTorProject/proteus/contents/{+path}","compare_url":"https://api.github.com/repos/TheTorProject/proteus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/TheTorProject/proteus/merges","archive_url":"https://api.github.com/repos/TheTorProject/proteus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/TheTorProject/proteus/downloads","issues_url":"https://api.github.com/repos/TheTorProject/proteus/issues{/number}","pulls_url":"https://api.github.com/repos/TheTorProject/proteus/pulls{/number}","milestones_url":"https://api.github.com/repos/TheTorProject/proteus/milestones{/number}","notifications_url":"https://api.github.com/repos/TheTorProject/proteus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/TheTorProject/proteus/labels{/name}","releases_url":"https://api.github.com/repos/TheTorProject/proteus/releases{/id}","deployments_url":"https://api.github.com/repos/TheTorProject/proteus/deployments","created_at":"2017-02-24T11:59:00Z","updated_at":"2017-08-06T20:49:23Z","pushed_at":"2017-12-05T19:07:37Z","git_url":"git://github.com/TheTorProject/proteus.git","ssh_url":"git@github.com:TheTorProject/proteus.git","clone_url":"https://github.com/TheTorProject/proteus.git","svn_url":"https://github.com/TheTorProject/proteus","homepage":null,"size":1851,"stargazers_count":5,"watchers_count":5,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":21,"license":{"key":"bsd-3-clause","name":"BSD 3-clause \"New\" or \"Revised\" License","spdx_id":"BSD-3-Clause","url":"https://api.github.com/licenses/bsd-3-clause"},"forks":5,"open_issues":21,"watchers":5,"default_branch":"master"}},"base":{"label":"TheTorProject:master","ref":"master","sha":"40cd6279973675d80a41df9e8293f4b0f80f5a06","user":{"login":"TheTorProject","id":3301912,"avatar_url":"https://avatars1.githubusercontent.com/u/3301912?v=4","gravatar_id":"","url":"https://api.github.com/users/TheTorProject","html_url":"https://github.com/TheTorProject","followers_url":"https://api.github.com/users/TheTorProject/followers","following_url":"https://api.github.com/users/TheTorProject/following{/other_user}","gists_url":"https://api.github.com/users/TheTorProject/gists{/gist_id}","starred_url":"https://api.github.com/users/TheTorProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TheTorProject/subscriptions","organizations_url":"https://api.github.com/users/TheTorProject/orgs","repos_url":"https://api.github.com/users/TheTorProject/repos","events_url":"https://api.github.com/users/TheTorProject/events{/privacy}","received_events_url":"https://api.github.com/users/TheTorProject/received_events","type":"Organization","site_admin":false},"repo":{"id":83037093,"name":"proteus","full_name":"TheTorProject/proteus","owner":{"login":"TheTorProject","id":3301912,"avatar_url":"https://avatars1.githubusercontent.com/u/3301912?v=4","gravatar_id":"","url":"https://api.github.com/users/TheTorProject","html_url":"https://github.com/TheTorProject","followers_url":"https://api.github.com/users/TheTorProject/followers","following_url":"https://api.github.com/users/TheTorProject/following{/other_user}","gists_url":"https://api.github.com/users/TheTorProject/gists{/gist_id}","starred_url":"https://api.github.com/users/TheTorProject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TheTorProject/subscriptions","organizations_url":"https://api.github.com/users/TheTorProject/orgs","repos_url":"https://api.github.com/users/TheTorProject/repos","events_url":"https://api.github.com/users/TheTorProject/events{/privacy}","received_events_url":"https://api.github.com/users/TheTorProject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/TheTorProject/proteus","description":"The OONI Probe Orchestration System","fork":false,"url":"https://api.github.com/repos/TheTorProject/proteus","forks_url":"https://api.github.com/repos/TheTorProject/proteus/forks","keys_url":"https://api.github.com/repos/TheTorProject/proteus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/TheTorProject/proteus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/TheTorProject/proteus/teams","hooks_url":"https://api.github.com/repos/TheTorProject/proteus/hooks","issue_events_url":"https://api.github.com/repos/TheTorProject/proteus/issues/events{/number}","events_url":"https://api.github.com/repos/TheTorProject/proteus/events","assignees_url":"https://api.github.com/repos/TheTorProject/proteus/assignees{/user}","branches_url":"https://api.github.com/repos/TheTorProject/proteus/branches{/branch}","tags_url":"https://api.github.com/repos/TheTorProject/proteus/tags","blobs_url":"https://api.github.com/repos/TheTorProject/proteus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/TheTorProject/proteus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/TheTorProject/proteus/git/refs{/sha}","trees_url":"https://api.github.com/repos/TheTorProject/proteus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/TheTorProject/proteus/statuses/{sha}","languages_url":"https://api.github.com/repos/TheTorProject/proteus/languages","stargazers_url":"https://api.github.com/repos/TheTorProject/proteus/stargazers","contributors_url":"https://api.github.com/repos/TheTorProject/proteus/contributors","subscribers_url":"https://api.github.com/repos/TheTorProject/proteus/subscribers","subscription_url":"https://api.github.com/repos/TheTorProject/proteus/subscription","commits_url":"https://api.github.com/repos/TheTorProject/proteus/commits{/sha}","git_commits_url":"https://api.github.com/repos/TheTorProject/proteus/git/commits{/sha}","comments_url":"https://api.github.com/repos/TheTorProject/proteus/comments{/number}","issue_comment_url":"https://api.github.com/repos/TheTorProject/proteus/issues/comments{/number}","contents_url":"https://api.github.com/repos/TheTorProject/proteus/contents/{+path}","compare_url":"https://api.github.com/repos/TheTorProject/proteus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/TheTorProject/proteus/merges","archive_url":"https://api.github.com/repos/TheTorProject/proteus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/TheTorProject/proteus/downloads","issues_url":"https://api.github.com/repos/TheTorProject/proteus/issues{/number}","pulls_url":"https://api.github.com/repos/TheTorProject/proteus/pulls{/number}","milestones_url":"https://api.github.com/repos/TheTorProject/proteus/milestones{/number}","notifications_url":"https://api.github.com/repos/TheTorProject/proteus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/TheTorProject/proteus/labels{/name}","releases_url":"https://api.github.com/repos/TheTorProject/proteus/releases{/id}","deployments_url":"https://api.github.com/repos/TheTorProject/proteus/deployments","created_at":"2017-02-24T11:59:00Z","updated_at":"2017-08-06T20:49:23Z","pushed_at":"2017-12-05T19:07:37Z","git_url":"git://github.com/TheTorProject/proteus.git","ssh_url":"git@github.com:TheTorProject/proteus.git","clone_url":"https://github.com/TheTorProject/proteus.git","svn_url":"https://github.com/TheTorProject/proteus","homepage":null,"size":1851,"stargazers_count":5,"watchers_count":5,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":21,"license":{"key":"bsd-3-clause","name":"BSD 3-clause \"New\" or \"Revised\" License","spdx_id":"BSD-3-Clause","url":"https://api.github.com/licenses/bsd-3-clause"},"forks":5,"open_issues":21,"watchers":5,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/TheTorProject/proteus/pulls/29"},"html":{"href":"https://github.com/TheTorProject/proteus/pull/29"},"issue":{"href":"https://api.github.com/repos/TheTorProject/proteus/issues/29"},"comments":{"href":"https://api.github.com/repos/TheTorProject/proteus/issues/29/comments"},"review_comments":{"href":"https://api.github.com/repos/TheTorProject/proteus/pulls/29/comments"},"review_comment":{"href":"https://api.github.com/repos/TheTorProject/proteus/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/TheTorProject/proteus/pulls/29/commits"},"statuses":{"href":"https://api.github.com/repos/TheTorProject/proteus/statuses/b1a47f556d941e62f5e8f083a3612123ed90d99e"}},"author_association":"NONE"}}
{"actor":{"display_login":"joelanders"}}