{"action":"created","comment":{"url":"https://api.github.com/repos/prometheus/prometheus/pulls/comments/105716171","pull_request_review_id":26593362,"id":105716171,"diff_hunk":"@@ -218,25 +253,181 @@ func (*QueueManager) NeedsThrottling() bool {\n // Start the queue manager sending samples to the remote storage.\n // Does not block.\n func (t *QueueManager) Start() {\n-\tfor i := 0; i < t.cfg.Shards; i++ {\n-\t\tgo t.runShard(i)\n-\t}\n+\tt.shardsMtx.Lock()\n+\tdefer t.shardsMtx.Unlock()\n+\tt.shards.start()\n }\n \n // Stop stops sending samples to the remote storage and waits for pending\n // sends to complete.\n func (t *QueueManager) Stop() {\n \tlog.Infof(\"Stopping remote storage...\")\n-\tfor _, shard := range t.shards {\n-\t\tclose(shard)\n-\t}\n+\tclose(t.quit)\n \tt.wg.Wait()\n+\tt.shardsMtx.Lock()\n+\tdefer t.shardsMtx.Unlock()\n+\tt.shards.stop()\n \tlog.Info(\"Remote storage stopped.\")\n }\n \n-func (t *QueueManager) runShard(i int) {\n+func (t *QueueManager) updateShardsLoop() {\n \tdefer t.wg.Done()\n-\tshard := t.shards[i]\n+\n+\tticker := time.Tick(shardUpdateDuration)\n+\tfor {\n+\t\tselect {\n+\t\tcase <-ticker:\n+\t\t\tt.caclulateDesiredShards()\n+\t\tcase <-t.quit:\n+\t\t\treturn\n+\t\t}\n+\t}\n+}\n+\n+func (t *QueueManager) caclulateDesiredShards() {\n+\tt.samplesIn.tick()\n+\tt.samplesOut.tick()\n+\tt.samplesOutDuration.tick()\n+\n+\t// We use the number of incoming samples as a prediction of how much work we\n+\t// will need to do next iteration.  We add to this any pending samples\n+\t// (received - send) so we can catch up with any backlog. We use the average\n+\t// outgoing batch latency to work out how how many shards we need.\n+\tvar (\n+\t\tsamplesIn          = t.samplesIn.rate()\n+\t\tsamplesOut         = t.samplesOut.rate()\n+\t\tsamplesPending     = samplesIn - samplesOut\n+\t\tsamplesOutDuration = t.samplesOutDuration.rate()\n+\t)\n+\n+\t// We use an integral accumulator, like in a PID, to help dampen oscillation.\n+\tt.integralAccumulator = t.integralAccumulator + (samplesPending * 0.1)\n+\n+\tif samplesOut <= 0 {\n+\t\treturn\n+\t}\n+\n+\tvar (\n+\t\ttimePerSample = samplesOutDuration / samplesOut\n+\t\tdesiredShards = (timePerSample * (samplesIn + samplesPending + t.integralAccumulator)) / float64(time.Second)\n+\t)\n+\tlog.Debugf(\"QueueManager.caclulateDesiredShards samplesIn=%f, samplesOut=%f, samplesPending=%f, desiredShards=%f\",\n+\t\tsamplesIn, samplesOut, samplesPending, desiredShards)\n+\n+\t// Changes in the number of shards must be greated than shardToleranceFraction.\n+\tvar (\n+\t\tlowerBound = float64(t.numShards) * (1. - shardToleranceFraction)\n+\t\tupperBound = float64(t.numShards) * (1. + shardToleranceFraction)\n+\t)\n+\tlog.Debugf(\"QueueManager.updateShardsLoop %f <= %f <= %f\", lowerBound, desiredShards, upperBound)\n+\tif lowerBound <= desiredShards && desiredShards <= upperBound {\n+\t\treturn\n+\t}\n+\n+\tnumShards := int(math.Ceil(desiredShards))\n+\tif numShards > t.cfg.MaxShards {\n+\t\tnumShards = t.cfg.MaxShards\n+\t}\n+\tif numShards == t.numShards {\n+\t\treturn\n+\t}\n+\n+\t// Resharding can take some time, and we want this loop\n+\t// to stay close to shardUpdateDuration.","path":"storage/remote/queue_manager.go","position":255,"original_position":255,"commit_id":"77cce900b87a95b79cea75da48a9cdc8d07b297a","original_commit_id":"77cce900b87a95b79cea75da48a9cdc8d07b297a","user":{"login":"juliusv","id":538008,"avatar_url":"https://avatars1.githubusercontent.com/u/538008?v=3","gravatar_id":"","url":"https://api.github.com/users/juliusv","html_url":"https://github.com/juliusv","followers_url":"https://api.github.com/users/juliusv/followers","following_url":"https://api.github.com/users/juliusv/following{/other_user}","gists_url":"https://api.github.com/users/juliusv/gists{/gist_id}","starred_url":"https://api.github.com/users/juliusv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/juliusv/subscriptions","organizations_url":"https://api.github.com/users/juliusv/orgs","repos_url":"https://api.github.com/users/juliusv/repos","events_url":"https://api.github.com/users/juliusv/events{/privacy}","received_events_url":"https://api.github.com/users/juliusv/received_events","type":"User","site_admin":false},"body":"The channel below can block though, if a previous resharding is still in progress... or is that so unlikely to happen that it's not going to make a noticeable difference for the rate update period and the resulting calculations?","created_at":"2017-03-13T17:09:36Z","updated_at":"2017-03-13T17:17:42Z","html_url":"https://github.com/prometheus/prometheus/pull/2494#discussion_r105716171","pull_request_url":"https://api.github.com/repos/prometheus/prometheus/pulls/2494","_links":{"self":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/comments/105716171"},"html":{"href":"https://github.com/prometheus/prometheus/pull/2494#discussion_r105716171"},"pull_request":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/2494"}}},"pull_request":{"url":"https://api.github.com/repos/prometheus/prometheus/pulls/2494","id":110418529,"html_url":"https://github.com/prometheus/prometheus/pull/2494","diff_url":"https://github.com/prometheus/prometheus/pull/2494.diff","patch_url":"https://github.com/prometheus/prometheus/pull/2494.patch","issue_url":"https://api.github.com/repos/prometheus/prometheus/issues/2494","number":2494,"state":"open","locked":false,"title":"Dynamically reshard the QueueManager based on observed load.","user":{"login":"tomwilkie","id":444037,"avatar_url":"https://avatars2.githubusercontent.com/u/444037?v=3","gravatar_id":"","url":"https://api.github.com/users/tomwilkie","html_url":"https://github.com/tomwilkie","followers_url":"https://api.github.com/users/tomwilkie/followers","following_url":"https://api.github.com/users/tomwilkie/following{/other_user}","gists_url":"https://api.github.com/users/tomwilkie/gists{/gist_id}","starred_url":"https://api.github.com/users/tomwilkie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomwilkie/subscriptions","organizations_url":"https://api.github.com/users/tomwilkie/orgs","repos_url":"https://api.github.com/users/tomwilkie/repos","events_url":"https://api.github.com/users/tomwilkie/events{/privacy}","received_events_url":"https://api.github.com/users/tomwilkie/received_events","type":"User","site_admin":false},"body":"Fixes #2445\r\n\r\nI tried using felixge/pidctrl, but ended up implementing a custom feedback loop that closely resembles a PID (minus the differentiated component).  It seems pretty stable in use.","created_at":"2017-03-13T14:40:44Z","updated_at":"2017-03-13T17:17:42Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c908fc848bfc27054eecf80b4644ff8a41354957","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/prometheus/prometheus/pulls/2494/commits","review_comments_url":"https://api.github.com/repos/prometheus/prometheus/pulls/2494/comments","review_comment_url":"https://api.github.com/repos/prometheus/prometheus/pulls/comments{/number}","comments_url":"https://api.github.com/repos/prometheus/prometheus/issues/2494/comments","statuses_url":"https://api.github.com/repos/prometheus/prometheus/statuses/77cce900b87a95b79cea75da48a9cdc8d07b297a","head":{"label":"tomwilkie:remote-write-sharding","ref":"remote-write-sharding","sha":"77cce900b87a95b79cea75da48a9cdc8d07b297a","user":{"login":"tomwilkie","id":444037,"avatar_url":"https://avatars2.githubusercontent.com/u/444037?v=3","gravatar_id":"","url":"https://api.github.com/users/tomwilkie","html_url":"https://github.com/tomwilkie","followers_url":"https://api.github.com/users/tomwilkie/followers","following_url":"https://api.github.com/users/tomwilkie/following{/other_user}","gists_url":"https://api.github.com/users/tomwilkie/gists{/gist_id}","starred_url":"https://api.github.com/users/tomwilkie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomwilkie/subscriptions","organizations_url":"https://api.github.com/users/tomwilkie/orgs","repos_url":"https://api.github.com/users/tomwilkie/repos","events_url":"https://api.github.com/users/tomwilkie/events{/privacy}","received_events_url":"https://api.github.com/users/tomwilkie/received_events","type":"User","site_admin":false},"repo":{"id":61650324,"name":"prometheus","full_name":"tomwilkie/prometheus","owner":{"login":"tomwilkie","id":444037,"avatar_url":"https://avatars2.githubusercontent.com/u/444037?v=3","gravatar_id":"","url":"https://api.github.com/users/tomwilkie","html_url":"https://github.com/tomwilkie","followers_url":"https://api.github.com/users/tomwilkie/followers","following_url":"https://api.github.com/users/tomwilkie/following{/other_user}","gists_url":"https://api.github.com/users/tomwilkie/gists{/gist_id}","starred_url":"https://api.github.com/users/tomwilkie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomwilkie/subscriptions","organizations_url":"https://api.github.com/users/tomwilkie/orgs","repos_url":"https://api.github.com/users/tomwilkie/repos","events_url":"https://api.github.com/users/tomwilkie/events{/privacy}","received_events_url":"https://api.github.com/users/tomwilkie/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tomwilkie/prometheus","description":"The Prometheus monitoring system and time series database.","fork":true,"url":"https://api.github.com/repos/tomwilkie/prometheus","forks_url":"https://api.github.com/repos/tomwilkie/prometheus/forks","keys_url":"https://api.github.com/repos/tomwilkie/prometheus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tomwilkie/prometheus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tomwilkie/prometheus/teams","hooks_url":"https://api.github.com/repos/tomwilkie/prometheus/hooks","issue_events_url":"https://api.github.com/repos/tomwilkie/prometheus/issues/events{/number}","events_url":"https://api.github.com/repos/tomwilkie/prometheus/events","assignees_url":"https://api.github.com/repos/tomwilkie/prometheus/assignees{/user}","branches_url":"https://api.github.com/repos/tomwilkie/prometheus/branches{/branch}","tags_url":"https://api.github.com/repos/tomwilkie/prometheus/tags","blobs_url":"https://api.github.com/repos/tomwilkie/prometheus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tomwilkie/prometheus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tomwilkie/prometheus/git/refs{/sha}","trees_url":"https://api.github.com/repos/tomwilkie/prometheus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tomwilkie/prometheus/statuses/{sha}","languages_url":"https://api.github.com/repos/tomwilkie/prometheus/languages","stargazers_url":"https://api.github.com/repos/tomwilkie/prometheus/stargazers","contributors_url":"https://api.github.com/repos/tomwilkie/prometheus/contributors","subscribers_url":"https://api.github.com/repos/tomwilkie/prometheus/subscribers","subscription_url":"https://api.github.com/repos/tomwilkie/prometheus/subscription","commits_url":"https://api.github.com/repos/tomwilkie/prometheus/commits{/sha}","git_commits_url":"https://api.github.com/repos/tomwilkie/prometheus/git/commits{/sha}","comments_url":"https://api.github.com/repos/tomwilkie/prometheus/comments{/number}","issue_comment_url":"https://api.github.com/repos/tomwilkie/prometheus/issues/comments{/number}","contents_url":"https://api.github.com/repos/tomwilkie/prometheus/contents/{+path}","compare_url":"https://api.github.com/repos/tomwilkie/prometheus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tomwilkie/prometheus/merges","archive_url":"https://api.github.com/repos/tomwilkie/prometheus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tomwilkie/prometheus/downloads","issues_url":"https://api.github.com/repos/tomwilkie/prometheus/issues{/number}","pulls_url":"https://api.github.com/repos/tomwilkie/prometheus/pulls{/number}","milestones_url":"https://api.github.com/repos/tomwilkie/prometheus/milestones{/number}","notifications_url":"https://api.github.com/repos/tomwilkie/prometheus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tomwilkie/prometheus/labels{/name}","releases_url":"https://api.github.com/repos/tomwilkie/prometheus/releases{/id}","deployments_url":"https://api.github.com/repos/tomwilkie/prometheus/deployments","created_at":"2016-06-21T16:41:01Z","updated_at":"2017-03-08T04:38:24Z","pushed_at":"2017-03-13T15:23:04Z","git_url":"git://github.com/tomwilkie/prometheus.git","ssh_url":"git@github.com:tomwilkie/prometheus.git","clone_url":"https://github.com/tomwilkie/prometheus.git","svn_url":"https://github.com/tomwilkie/prometheus","homepage":"https://prometheus.io/","size":17965,"stargazers_count":3,"watchers_count":3,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":3,"default_branch":"master"}},"base":{"label":"prometheus:master","ref":"master","sha":"5aa90c075b7ea4f13292b9bd7d753de3145a39dd","user":{"login":"prometheus","id":3380462,"avatar_url":"https://avatars0.githubusercontent.com/u/3380462?v=3","gravatar_id":"","url":"https://api.github.com/users/prometheus","html_url":"https://github.com/prometheus","followers_url":"https://api.github.com/users/prometheus/followers","following_url":"https://api.github.com/users/prometheus/following{/other_user}","gists_url":"https://api.github.com/users/prometheus/gists{/gist_id}","starred_url":"https://api.github.com/users/prometheus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prometheus/subscriptions","organizations_url":"https://api.github.com/users/prometheus/orgs","repos_url":"https://api.github.com/users/prometheus/repos","events_url":"https://api.github.com/users/prometheus/events{/privacy}","received_events_url":"https://api.github.com/users/prometheus/received_events","type":"Organization","site_admin":false},"repo":{"id":6838921,"name":"prometheus","full_name":"prometheus/prometheus","owner":{"login":"prometheus","id":3380462,"avatar_url":"https://avatars0.githubusercontent.com/u/3380462?v=3","gravatar_id":"","url":"https://api.github.com/users/prometheus","html_url":"https://github.com/prometheus","followers_url":"https://api.github.com/users/prometheus/followers","following_url":"https://api.github.com/users/prometheus/following{/other_user}","gists_url":"https://api.github.com/users/prometheus/gists{/gist_id}","starred_url":"https://api.github.com/users/prometheus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prometheus/subscriptions","organizations_url":"https://api.github.com/users/prometheus/orgs","repos_url":"https://api.github.com/users/prometheus/repos","events_url":"https://api.github.com/users/prometheus/events{/privacy}","received_events_url":"https://api.github.com/users/prometheus/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/prometheus/prometheus","description":"The Prometheus monitoring system and time series database.","fork":false,"url":"https://api.github.com/repos/prometheus/prometheus","forks_url":"https://api.github.com/repos/prometheus/prometheus/forks","keys_url":"https://api.github.com/repos/prometheus/prometheus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/prometheus/prometheus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/prometheus/prometheus/teams","hooks_url":"https://api.github.com/repos/prometheus/prometheus/hooks","issue_events_url":"https://api.github.com/repos/prometheus/prometheus/issues/events{/number}","events_url":"https://api.github.com/repos/prometheus/prometheus/events","assignees_url":"https://api.github.com/repos/prometheus/prometheus/assignees{/user}","branches_url":"https://api.github.com/repos/prometheus/prometheus/branches{/branch}","tags_url":"https://api.github.com/repos/prometheus/prometheus/tags","blobs_url":"https://api.github.com/repos/prometheus/prometheus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/prometheus/prometheus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/prometheus/prometheus/git/refs{/sha}","trees_url":"https://api.github.com/repos/prometheus/prometheus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/prometheus/prometheus/statuses/{sha}","languages_url":"https://api.github.com/repos/prometheus/prometheus/languages","stargazers_url":"https://api.github.com/repos/prometheus/prometheus/stargazers","contributors_url":"https://api.github.com/repos/prometheus/prometheus/contributors","subscribers_url":"https://api.github.com/repos/prometheus/prometheus/subscribers","subscription_url":"https://api.github.com/repos/prometheus/prometheus/subscription","commits_url":"https://api.github.com/repos/prometheus/prometheus/commits{/sha}","git_commits_url":"https://api.github.com/repos/prometheus/prometheus/git/commits{/sha}","comments_url":"https://api.github.com/repos/prometheus/prometheus/comments{/number}","issue_comment_url":"https://api.github.com/repos/prometheus/prometheus/issues/comments{/number}","contents_url":"https://api.github.com/repos/prometheus/prometheus/contents/{+path}","compare_url":"https://api.github.com/repos/prometheus/prometheus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/prometheus/prometheus/merges","archive_url":"https://api.github.com/repos/prometheus/prometheus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/prometheus/prometheus/downloads","issues_url":"https://api.github.com/repos/prometheus/prometheus/issues{/number}","pulls_url":"https://api.github.com/repos/prometheus/prometheus/pulls{/number}","milestones_url":"https://api.github.com/repos/prometheus/prometheus/milestones{/number}","notifications_url":"https://api.github.com/repos/prometheus/prometheus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/prometheus/prometheus/labels{/name}","releases_url":"https://api.github.com/repos/prometheus/prometheus/releases{/id}","deployments_url":"https://api.github.com/repos/prometheus/prometheus/deployments","created_at":"2012-11-24T11:14:12Z","updated_at":"2017-03-13T14:57:42Z","pushed_at":"2017-03-13T15:23:06Z","git_url":"git://github.com/prometheus/prometheus.git","ssh_url":"git@github.com:prometheus/prometheus.git","clone_url":"https://github.com/prometheus/prometheus.git","svn_url":"https://github.com/prometheus/prometheus","homepage":"https://prometheus.io/","size":20861,"stargazers_count":8521,"watchers_count":8521,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":861,"mirror_url":null,"open_issues_count":227,"forks":861,"open_issues":227,"watchers":8521,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/2494"},"html":{"href":"https://github.com/prometheus/prometheus/pull/2494"},"issue":{"href":"https://api.github.com/repos/prometheus/prometheus/issues/2494"},"comments":{"href":"https://api.github.com/repos/prometheus/prometheus/issues/2494/comments"},"review_comments":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/2494/comments"},"review_comment":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/2494/commits"},"statuses":{"href":"https://api.github.com/repos/prometheus/prometheus/statuses/77cce900b87a95b79cea75da48a9cdc8d07b297a"}}}}
{"actor":{"display_login":"juliusv"}}