{"action":"created","comment":{"url":"https://api.github.com/repos/raintank/metrictank/pulls/comments/101920879","pull_request_review_id":22662324,"id":101920879,"diff_hunk":"@@ -0,0 +1,165 @@\n+package main\n+\n+import (\n+\t\"compress/gzip\"\n+\t\"encoding/json\"\n+\t\"flag\"\n+\t\"fmt\"\n+\t\"io\"\n+\t\"io/ioutil\"\n+\t\"math\"\n+\t\"net/http\"\n+\t\"strings\"\n+\t\"time\"\n+\n+\t\"github.com/Shopify/sarama\"\n+\t\"github.com/gocql/gocql\"\n+\t\"github.com/raintank/dur\"\n+\t\"github.com/raintank/metrictank/cmd/whisperImporter\"\n+\t\"github.com/raintank/metrictank/idx\"\n+\t\"github.com/raintank/metrictank/idx/cassandra\"\n+\t\"github.com/raintank/metrictank/mdata\"\n+\t\"gopkg.in/raintank/schema.v1\"\n+)\n+\n+const table_name_format = `metric_%d`\n+\n+var (\n+\thttpEndpoint      = flag.String(\"http-endpoint\", \"127.0.0.1:8080\", \"The http endpoint to listen on\")\n+\tthreads           = flag.Int(\"threads\", 10, \"Number of workers threads to process .wsp files\")\n+\texitOnError       = flag.Bool(\"exit-on-error\", true, \"Exit with a message when there's an error\")\n+\tcassandraAddrs    = flag.String(\"cassandra-addrs\", \"localhost\", \"cassandra host (may be given multiple times as comma-separated list)\")\n+\tcassandraKeyspace = flag.String(\"cassandra-keyspace\", \"metrictank\", \"cassandra keyspace to use for storing the metric data table\")\n+\tcassAddr          = flag.String(\"cass-addr\", \"localhost\", \"Address of cassandra host.\")\n+\tkeyspace          = flag.String(\"keyspace\", \"raintank\", \"Cassandra keyspace to use.\")\n+\tttlsStr           = flag.String(\"ttls\", \"35d\", \"list of ttl strings used by MT separated by :\")\n+\twindowFactor      = flag.Int(\"window-factor\", 20, \"the window factor be used when creating the metric table schema\")\n+\tpartitionBy       = flag.String(\"partition-by\", \"bySeries\", \"Method used for paritioning metrics. (byOrg|bySeries)\")\n+\tnumPartitions     = flag.Int(\"num-partitions\", 0, \"Number of Partitions\")\n+)\n+\n+func throwError(msg string) {\n+\tpanic(msg)\n+}\n+\n+type Server struct {\n+\tCluster     *gocql.ClusterConfig\n+\tTTLTables   mdata.TTLTables\n+\tPartitioner sarama.Partitioner\n+\tIndex       idx.MetricIndex\n+}\n+\n+func (s *Server) getPartition(md *schema.MetricData) int32 {\n+\tvar key []byte\n+\tswitch *partitionBy {\n+\tcase \"byOrg\":\n+\t\tkey = md.KeyByOrgId(nil)\n+\tcase \"bySeries\":\n+\t\tkey = md.KeyBySeries(nil)\n+\t}\n+\tpartition, _ := s.Partitioner.Partition(&sarama.ProducerMessage{Key: sarama.ByteEncoder(key)}, int32(*numPartitions))\n+\treturn partition","path":"cmd/whisperImporter/chunkWriter/main.go","position":61,"original_position":61,"commit_id":"41a60af558838c20dd42f27e7bde5f5ae2403db4","original_commit_id":"41a60af558838c20dd42f27e7bde5f5ae2403db4","user":{"login":"Dieterbe","id":20774,"avatar_url":"https://avatars.githubusercontent.com/u/20774?v=3","gravatar_id":"","url":"https://api.github.com/users/Dieterbe","html_url":"https://github.com/Dieterbe","followers_url":"https://api.github.com/users/Dieterbe/followers","following_url":"https://api.github.com/users/Dieterbe/following{/other_user}","gists_url":"https://api.github.com/users/Dieterbe/gists{/gist_id}","starred_url":"https://api.github.com/users/Dieterbe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Dieterbe/subscriptions","organizations_url":"https://api.github.com/users/Dieterbe/orgs","repos_url":"https://api.github.com/users/Dieterbe/repos","events_url":"https://api.github.com/users/Dieterbe/events{/privacy}","received_events_url":"https://api.github.com/users/Dieterbe/received_events","type":"User","site_admin":false},"body":"Why not just use the partitioner in MT, e.g. `cluster/partitioner.Kafka` ?","created_at":"2017-02-19T15:41:25Z","updated_at":"2017-02-19T15:41:25Z","html_url":"https://github.com/raintank/metrictank/pull/533#discussion_r101920879","pull_request_url":"https://api.github.com/repos/raintank/metrictank/pulls/533","_links":{"self":{"href":"https://api.github.com/repos/raintank/metrictank/pulls/comments/101920879"},"html":{"href":"https://github.com/raintank/metrictank/pull/533#discussion_r101920879"},"pull_request":{"href":"https://api.github.com/repos/raintank/metrictank/pulls/533"}}},"pull_request":{"url":"https://api.github.com/repos/raintank/metrictank/pulls/533","id":106625443,"html_url":"https://github.com/raintank/metrictank/pull/533","diff_url":"https://github.com/raintank/metrictank/pull/533.diff","patch_url":"https://github.com/raintank/metrictank/pull/533.patch","issue_url":"https://api.github.com/repos/raintank/metrictank/issues/533","number":533,"state":"open","locked":false,"title":"[WIP] Whisper importer","user":{"login":"replay","id":195371,"avatar_url":"https://avatars.githubusercontent.com/u/195371?v=3","gravatar_id":"","url":"https://api.github.com/users/replay","html_url":"https://github.com/replay","followers_url":"https://api.github.com/users/replay/followers","following_url":"https://api.github.com/users/replay/following{/other_user}","gists_url":"https://api.github.com/users/replay/gists{/gist_id}","starred_url":"https://api.github.com/users/replay/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/replay/subscriptions","organizations_url":"https://api.github.com/users/replay/orgs","repos_url":"https://api.github.com/users/replay/repos","events_url":"https://api.github.com/users/replay/events{/privacy}","received_events_url":"https://api.github.com/users/replay/received_events","type":"User","site_admin":false},"body":"Adds two tools to:\r\n1) Read whisper files, generate chunks out of the and post them into an HTTP endpoint as gzipped JSON\r\n2) Listen on an HTTP endpoint, read and unpack gzipped JSON and store the retrieved chunks into cassandra","created_at":"2017-02-16T22:10:37Z","updated_at":"2017-02-19T15:41:25Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a3d4f9d7589e9fbe53568662fbe71f318e93b448","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/raintank/metrictank/pulls/533/commits","review_comments_url":"https://api.github.com/repos/raintank/metrictank/pulls/533/comments","review_comment_url":"https://api.github.com/repos/raintank/metrictank/pulls/comments{/number}","comments_url":"https://api.github.com/repos/raintank/metrictank/issues/533/comments","statuses_url":"https://api.github.com/repos/raintank/metrictank/statuses/41a60af558838c20dd42f27e7bde5f5ae2403db4","head":{"label":"raintank:whisper-importer","ref":"whisper-importer","sha":"41a60af558838c20dd42f27e7bde5f5ae2403db4","user":{"login":"raintank","id":8512431,"avatar_url":"https://avatars.githubusercontent.com/u/8512431?v=3","gravatar_id":"","url":"https://api.github.com/users/raintank","html_url":"https://github.com/raintank","followers_url":"https://api.github.com/users/raintank/followers","following_url":"https://api.github.com/users/raintank/following{/other_user}","gists_url":"https://api.github.com/users/raintank/gists{/gist_id}","starred_url":"https://api.github.com/users/raintank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/raintank/subscriptions","organizations_url":"https://api.github.com/users/raintank/orgs","repos_url":"https://api.github.com/users/raintank/repos","events_url":"https://api.github.com/users/raintank/events{/privacy}","received_events_url":"https://api.github.com/users/raintank/received_events","type":"Organization","site_admin":false},"repo":{"id":30401116,"name":"metrictank","full_name":"raintank/metrictank","owner":{"login":"raintank","id":8512431,"avatar_url":"https://avatars.githubusercontent.com/u/8512431?v=3","gravatar_id":"","url":"https://api.github.com/users/raintank","html_url":"https://github.com/raintank","followers_url":"https://api.github.com/users/raintank/followers","following_url":"https://api.github.com/users/raintank/following{/other_user}","gists_url":"https://api.github.com/users/raintank/gists{/gist_id}","starred_url":"https://api.github.com/users/raintank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/raintank/subscriptions","organizations_url":"https://api.github.com/users/raintank/orgs","repos_url":"https://api.github.com/users/raintank/repos","events_url":"https://api.github.com/users/raintank/events{/privacy}","received_events_url":"https://api.github.com/users/raintank/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/raintank/metrictank","description":"Cassandra-backed, metrics2.0 based, multi-tenant timeseries database for Graphite and friends","fork":false,"url":"https://api.github.com/repos/raintank/metrictank","forks_url":"https://api.github.com/repos/raintank/metrictank/forks","keys_url":"https://api.github.com/repos/raintank/metrictank/keys{/key_id}","collaborators_url":"https://api.github.com/repos/raintank/metrictank/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/raintank/metrictank/teams","hooks_url":"https://api.github.com/repos/raintank/metrictank/hooks","issue_events_url":"https://api.github.com/repos/raintank/metrictank/issues/events{/number}","events_url":"https://api.github.com/repos/raintank/metrictank/events","assignees_url":"https://api.github.com/repos/raintank/metrictank/assignees{/user}","branches_url":"https://api.github.com/repos/raintank/metrictank/branches{/branch}","tags_url":"https://api.github.com/repos/raintank/metrictank/tags","blobs_url":"https://api.github.com/repos/raintank/metrictank/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/raintank/metrictank/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/raintank/metrictank/git/refs{/sha}","trees_url":"https://api.github.com/repos/raintank/metrictank/git/trees{/sha}","statuses_url":"https://api.github.com/repos/raintank/metrictank/statuses/{sha}","languages_url":"https://api.github.com/repos/raintank/metrictank/languages","stargazers_url":"https://api.github.com/repos/raintank/metrictank/stargazers","contributors_url":"https://api.github.com/repos/raintank/metrictank/contributors","subscribers_url":"https://api.github.com/repos/raintank/metrictank/subscribers","subscription_url":"https://api.github.com/repos/raintank/metrictank/subscription","commits_url":"https://api.github.com/repos/raintank/metrictank/commits{/sha}","git_commits_url":"https://api.github.com/repos/raintank/metrictank/git/commits{/sha}","comments_url":"https://api.github.com/repos/raintank/metrictank/comments{/number}","issue_comment_url":"https://api.github.com/repos/raintank/metrictank/issues/comments{/number}","contents_url":"https://api.github.com/repos/raintank/metrictank/contents/{+path}","compare_url":"https://api.github.com/repos/raintank/metrictank/compare/{base}...{head}","merges_url":"https://api.github.com/repos/raintank/metrictank/merges","archive_url":"https://api.github.com/repos/raintank/metrictank/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/raintank/metrictank/downloads","issues_url":"https://api.github.com/repos/raintank/metrictank/issues{/number}","pulls_url":"https://api.github.com/repos/raintank/metrictank/pulls{/number}","milestones_url":"https://api.github.com/repos/raintank/metrictank/milestones{/number}","notifications_url":"https://api.github.com/repos/raintank/metrictank/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/raintank/metrictank/labels{/name}","releases_url":"https://api.github.com/repos/raintank/metrictank/releases{/id}","deployments_url":"https://api.github.com/repos/raintank/metrictank/deployments","created_at":"2015-02-06T07:44:12Z","updated_at":"2017-02-16T16:04:55Z","pushed_at":"2017-02-17T23:39:59Z","git_url":"git://github.com/raintank/metrictank.git","ssh_url":"git@github.com:raintank/metrictank.git","clone_url":"https://github.com/raintank/metrictank.git","svn_url":"https://github.com/raintank/metrictank","homepage":"","size":10174,"stargazers_count":97,"watchers_count":97,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":10,"mirror_url":null,"open_issues_count":88,"forks":10,"open_issues":88,"watchers":97,"default_branch":"master"}},"base":{"label":"raintank:master","ref":"master","sha":"02041d6c04eef7db8140b883591ee322b8796fdc","user":{"login":"raintank","id":8512431,"avatar_url":"https://avatars.githubusercontent.com/u/8512431?v=3","gravatar_id":"","url":"https://api.github.com/users/raintank","html_url":"https://github.com/raintank","followers_url":"https://api.github.com/users/raintank/followers","following_url":"https://api.github.com/users/raintank/following{/other_user}","gists_url":"https://api.github.com/users/raintank/gists{/gist_id}","starred_url":"https://api.github.com/users/raintank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/raintank/subscriptions","organizations_url":"https://api.github.com/users/raintank/orgs","repos_url":"https://api.github.com/users/raintank/repos","events_url":"https://api.github.com/users/raintank/events{/privacy}","received_events_url":"https://api.github.com/users/raintank/received_events","type":"Organization","site_admin":false},"repo":{"id":30401116,"name":"metrictank","full_name":"raintank/metrictank","owner":{"login":"raintank","id":8512431,"avatar_url":"https://avatars.githubusercontent.com/u/8512431?v=3","gravatar_id":"","url":"https://api.github.com/users/raintank","html_url":"https://github.com/raintank","followers_url":"https://api.github.com/users/raintank/followers","following_url":"https://api.github.com/users/raintank/following{/other_user}","gists_url":"https://api.github.com/users/raintank/gists{/gist_id}","starred_url":"https://api.github.com/users/raintank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/raintank/subscriptions","organizations_url":"https://api.github.com/users/raintank/orgs","repos_url":"https://api.github.com/users/raintank/repos","events_url":"https://api.github.com/users/raintank/events{/privacy}","received_events_url":"https://api.github.com/users/raintank/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/raintank/metrictank","description":"Cassandra-backed, metrics2.0 based, multi-tenant timeseries database for Graphite and friends","fork":false,"url":"https://api.github.com/repos/raintank/metrictank","forks_url":"https://api.github.com/repos/raintank/metrictank/forks","keys_url":"https://api.github.com/repos/raintank/metrictank/keys{/key_id}","collaborators_url":"https://api.github.com/repos/raintank/metrictank/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/raintank/metrictank/teams","hooks_url":"https://api.github.com/repos/raintank/metrictank/hooks","issue_events_url":"https://api.github.com/repos/raintank/metrictank/issues/events{/number}","events_url":"https://api.github.com/repos/raintank/metrictank/events","assignees_url":"https://api.github.com/repos/raintank/metrictank/assignees{/user}","branches_url":"https://api.github.com/repos/raintank/metrictank/branches{/branch}","tags_url":"https://api.github.com/repos/raintank/metrictank/tags","blobs_url":"https://api.github.com/repos/raintank/metrictank/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/raintank/metrictank/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/raintank/metrictank/git/refs{/sha}","trees_url":"https://api.github.com/repos/raintank/metrictank/git/trees{/sha}","statuses_url":"https://api.github.com/repos/raintank/metrictank/statuses/{sha}","languages_url":"https://api.github.com/repos/raintank/metrictank/languages","stargazers_url":"https://api.github.com/repos/raintank/metrictank/stargazers","contributors_url":"https://api.github.com/repos/raintank/metrictank/contributors","subscribers_url":"https://api.github.com/repos/raintank/metrictank/subscribers","subscription_url":"https://api.github.com/repos/raintank/metrictank/subscription","commits_url":"https://api.github.com/repos/raintank/metrictank/commits{/sha}","git_commits_url":"https://api.github.com/repos/raintank/metrictank/git/commits{/sha}","comments_url":"https://api.github.com/repos/raintank/metrictank/comments{/number}","issue_comment_url":"https://api.github.com/repos/raintank/metrictank/issues/comments{/number}","contents_url":"https://api.github.com/repos/raintank/metrictank/contents/{+path}","compare_url":"https://api.github.com/repos/raintank/metrictank/compare/{base}...{head}","merges_url":"https://api.github.com/repos/raintank/metrictank/merges","archive_url":"https://api.github.com/repos/raintank/metrictank/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/raintank/metrictank/downloads","issues_url":"https://api.github.com/repos/raintank/metrictank/issues{/number}","pulls_url":"https://api.github.com/repos/raintank/metrictank/pulls{/number}","milestones_url":"https://api.github.com/repos/raintank/metrictank/milestones{/number}","notifications_url":"https://api.github.com/repos/raintank/metrictank/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/raintank/metrictank/labels{/name}","releases_url":"https://api.github.com/repos/raintank/metrictank/releases{/id}","deployments_url":"https://api.github.com/repos/raintank/metrictank/deployments","created_at":"2015-02-06T07:44:12Z","updated_at":"2017-02-16T16:04:55Z","pushed_at":"2017-02-17T23:39:59Z","git_url":"git://github.com/raintank/metrictank.git","ssh_url":"git@github.com:raintank/metrictank.git","clone_url":"https://github.com/raintank/metrictank.git","svn_url":"https://github.com/raintank/metrictank","homepage":"","size":10174,"stargazers_count":97,"watchers_count":97,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":10,"mirror_url":null,"open_issues_count":88,"forks":10,"open_issues":88,"watchers":97,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/raintank/metrictank/pulls/533"},"html":{"href":"https://github.com/raintank/metrictank/pull/533"},"issue":{"href":"https://api.github.com/repos/raintank/metrictank/issues/533"},"comments":{"href":"https://api.github.com/repos/raintank/metrictank/issues/533/comments"},"review_comments":{"href":"https://api.github.com/repos/raintank/metrictank/pulls/533/comments"},"review_comment":{"href":"https://api.github.com/repos/raintank/metrictank/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/raintank/metrictank/pulls/533/commits"},"statuses":{"href":"https://api.github.com/repos/raintank/metrictank/statuses/41a60af558838c20dd42f27e7bde5f5ae2403db4"}}}}
{"actor":{"display_login":"Dieterbe"}}