{"action":"created","comment":{"url":"https://api.github.com/repos/m3db/m3db/pulls/comments/129990900","pull_request_review_id":52820930,"id":129990900,"diff_hunk":"@@ -159,15 +157,60 @@ func (m *cleanupManager) commitLogTimes(t time.Time) (time.Time, []time.Time) {\n \t// data loss.\n \n \tcandidateTimes := timesInRange(earliest, latest, blockSize)\n+\tnamespaces := m.database.GetOwnedNamespaces()\n \tcleanupTimes := filterTimes(candidateTimes, func(t time.Time) bool {\n-\t\tleftBlockStart, rightBlockStart := t.Add(-blockSize), t.Add(blockSize)\n-\t\tanyPendingFlushes := m.fm.NeedsFlush(leftBlockStart, rightBlockStart)\n-\t\treturn !anyPendingFlushes\n+\t\tfor _, ns := range namespaces {\n+\t\t\tropts := ns.Options().RetentionOptions()\n+\t\t\tstart, end := commitLogNamespaceBlockTimes(t, blockSize, ropts)\n+\t\t\tif ns.NeedsFlush(start, end) {\n+\t\t\t\treturn false\n+\t\t\t}\n+\t\t}\n+\t\treturn true\n \t})\n \n \treturn earliest, cleanupTimes\n }\n \n+// commitLogNamespaceBlockTimes returns the range of namespace block starts which for which the\n+// given commit log block may contain data for.\n+//\n+// consider the situation where we have a single namespace, and a commit log with the following\n+// retention options:\n+//             buffer past | buffer future | block size\n+// namespace:    ns_bp     |     ns_bf     |    ns_bs\n+// commit log:     _       |      _        |    cl_bs\n+//\n+// for the commit log block with start time `t`, we can receive data for a range of namespace\n+// blocks depending on the namespace retention options. The range is given by these relationships:\n+//\t- earliest ns block start = t.Add(-ns_bp).Truncate(ns_bs)\n+//  - latest ns block start   = t.Add(cl_bs).Add(ns_bf).Ceil(ns_bs)\n+// NB:\n+// - blockStart assumed to be aligned to commit log block size\n+func commitLogNamespaceBlockTimes(\n+\tblockStart time.Time,\n+\tcommitlogBlockSize time.Duration,\n+\tnsRetention retention.Options,\n+) (time.Time, time.Time) {\n+\tearliest := blockStart.Add(-nsRetention.BufferPast()).Truncate(nsRetention.BlockSize())\n+\tlatest := blockStart.Add(commitlogBlockSize).Add(nsRetention.BufferFuture())\n+\tlatest = ceilDate(latest).ceil(nsRetention.BlockSize())","path":"storage/cleanup.go","position":null,"original_position":77,"commit_id":"02e877740f616c0710ecbc759f6098e4d7af8e99","original_commit_id":"0a245d3e9a325386211f9c1c77d4fedae5569de5","user":{"login":"prateek","id":199982,"avatar_url":"https://avatars3.githubusercontent.com/u/199982?v=4","gravatar_id":"","url":"https://api.github.com/users/prateek","html_url":"https://github.com/prateek","followers_url":"https://api.github.com/users/prateek/followers","following_url":"https://api.github.com/users/prateek/following{/other_user}","gists_url":"https://api.github.com/users/prateek/gists{/gist_id}","starred_url":"https://api.github.com/users/prateek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prateek/subscriptions","organizations_url":"https://api.github.com/users/prateek/orgs","repos_url":"https://api.github.com/users/prateek/repos","events_url":"https://api.github.com/users/prateek/events{/privacy}","received_events_url":"https://api.github.com/users/prateek/received_events","type":"User","site_admin":false},"body":"done","created_at":"2017-07-28T00:36:30Z","updated_at":"2017-07-28T00:36:30Z","html_url":"https://github.com/m3db/m3db/pull/321#discussion_r129990900","pull_request_url":"https://api.github.com/repos/m3db/m3db/pulls/321","_links":{"self":{"href":"https://api.github.com/repos/m3db/m3db/pulls/comments/129990900"},"html":{"href":"https://github.com/m3db/m3db/pull/321#discussion_r129990900"},"pull_request":{"href":"https://api.github.com/repos/m3db/m3db/pulls/321"}}},"pull_request":{"url":"https://api.github.com/repos/m3db/m3db/pulls/321","id":132642731,"html_url":"https://github.com/m3db/m3db/pull/321","diff_url":"https://github.com/m3db/m3db/pull/321.diff","patch_url":"https://github.com/m3db/m3db/pull/321.patch","issue_url":"https://api.github.com/repos/m3db/m3db/issues/321","number":321,"state":"open","locked":false,"title":"Rework CommitLog cleanup times","user":{"login":"prateek","id":199982,"avatar_url":"https://avatars3.githubusercontent.com/u/199982?v=4","gravatar_id":"","url":"https://api.github.com/users/prateek","html_url":"https://github.com/prateek","followers_url":"https://api.github.com/users/prateek/followers","following_url":"https://api.github.com/users/prateek/following{/other_user}","gists_url":"https://api.github.com/users/prateek/gists{/gist_id}","starred_url":"https://api.github.com/users/prateek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prateek/subscriptions","organizations_url":"https://api.github.com/users/prateek/orgs","repos_url":"https://api.github.com/users/prateek/repos","events_url":"https://api.github.com/users/prateek/events{/privacy}","received_events_url":"https://api.github.com/users/prateek/received_events","type":"User","site_admin":false},"body":"Changes:\r\n- Rework commit log cleanup times calculation\r\n- Add property test to ensure we don't cleanup commit logs for un-flushed data\r\n\r\n/cc @xichen2020 @robskillington \r\n","created_at":"2017-07-27T00:27:41Z","updated_at":"2017-07-28T00:36:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"28434bf53a794d67bfd37a76560a55cf612f7436","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/m3db/m3db/pulls/321/commits","review_comments_url":"https://api.github.com/repos/m3db/m3db/pulls/321/comments","review_comment_url":"https://api.github.com/repos/m3db/m3db/pulls/comments{/number}","comments_url":"https://api.github.com/repos/m3db/m3db/issues/321/comments","statuses_url":"https://api.github.com/repos/m3db/m3db/statuses/02e877740f616c0710ecbc759f6098e4d7af8e99","head":{"label":"m3db:prateek/prop-testing-commitlog-cleanup","ref":"prateek/prop-testing-commitlog-cleanup","sha":"02e877740f616c0710ecbc759f6098e4d7af8e99","user":{"login":"m3db","id":19937171,"avatar_url":"https://avatars0.githubusercontent.com/u/19937171?v=4","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"repo":{"id":61153677,"name":"m3db","full_name":"m3db/m3db","owner":{"login":"m3db","id":19937171,"avatar_url":"https://avatars0.githubusercontent.com/u/19937171?v=4","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/m3db/m3db","description":"A distributed time series database using M3TSZ compression ","fork":false,"url":"https://api.github.com/repos/m3db/m3db","forks_url":"https://api.github.com/repos/m3db/m3db/forks","keys_url":"https://api.github.com/repos/m3db/m3db/keys{/key_id}","collaborators_url":"https://api.github.com/repos/m3db/m3db/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/m3db/m3db/teams","hooks_url":"https://api.github.com/repos/m3db/m3db/hooks","issue_events_url":"https://api.github.com/repos/m3db/m3db/issues/events{/number}","events_url":"https://api.github.com/repos/m3db/m3db/events","assignees_url":"https://api.github.com/repos/m3db/m3db/assignees{/user}","branches_url":"https://api.github.com/repos/m3db/m3db/branches{/branch}","tags_url":"https://api.github.com/repos/m3db/m3db/tags","blobs_url":"https://api.github.com/repos/m3db/m3db/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/m3db/m3db/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/m3db/m3db/git/refs{/sha}","trees_url":"https://api.github.com/repos/m3db/m3db/git/trees{/sha}","statuses_url":"https://api.github.com/repos/m3db/m3db/statuses/{sha}","languages_url":"https://api.github.com/repos/m3db/m3db/languages","stargazers_url":"https://api.github.com/repos/m3db/m3db/stargazers","contributors_url":"https://api.github.com/repos/m3db/m3db/contributors","subscribers_url":"https://api.github.com/repos/m3db/m3db/subscribers","subscription_url":"https://api.github.com/repos/m3db/m3db/subscription","commits_url":"https://api.github.com/repos/m3db/m3db/commits{/sha}","git_commits_url":"https://api.github.com/repos/m3db/m3db/git/commits{/sha}","comments_url":"https://api.github.com/repos/m3db/m3db/comments{/number}","issue_comment_url":"https://api.github.com/repos/m3db/m3db/issues/comments{/number}","contents_url":"https://api.github.com/repos/m3db/m3db/contents/{+path}","compare_url":"https://api.github.com/repos/m3db/m3db/compare/{base}...{head}","merges_url":"https://api.github.com/repos/m3db/m3db/merges","archive_url":"https://api.github.com/repos/m3db/m3db/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/m3db/m3db/downloads","issues_url":"https://api.github.com/repos/m3db/m3db/issues{/number}","pulls_url":"https://api.github.com/repos/m3db/m3db/pulls{/number}","milestones_url":"https://api.github.com/repos/m3db/m3db/milestones{/number}","notifications_url":"https://api.github.com/repos/m3db/m3db/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/m3db/m3db/labels{/name}","releases_url":"https://api.github.com/repos/m3db/m3db/releases{/id}","deployments_url":"https://api.github.com/repos/m3db/m3db/deployments","created_at":"2016-06-14T20:29:45Z","updated_at":"2017-07-14T05:20:41Z","pushed_at":"2017-07-28T00:35:42Z","git_url":"git://github.com/m3db/m3db.git","ssh_url":"git@github.com:m3db/m3db.git","clone_url":"https://github.com/m3db/m3db.git","svn_url":"https://github.com/m3db/m3db","homepage":"http://uber.github.io/","size":4430,"stargazers_count":110,"watchers_count":110,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":20,"forks":2,"open_issues":20,"watchers":110,"default_branch":"master"}},"base":{"label":"m3db:prateek/ns-isolation","ref":"prateek/ns-isolation","sha":"3d07c74e6e6aec7951632208ebe3ccbfb4d01384","user":{"login":"m3db","id":19937171,"avatar_url":"https://avatars0.githubusercontent.com/u/19937171?v=4","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"repo":{"id":61153677,"name":"m3db","full_name":"m3db/m3db","owner":{"login":"m3db","id":19937171,"avatar_url":"https://avatars0.githubusercontent.com/u/19937171?v=4","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/m3db/m3db","description":"A distributed time series database using M3TSZ compression ","fork":false,"url":"https://api.github.com/repos/m3db/m3db","forks_url":"https://api.github.com/repos/m3db/m3db/forks","keys_url":"https://api.github.com/repos/m3db/m3db/keys{/key_id}","collaborators_url":"https://api.github.com/repos/m3db/m3db/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/m3db/m3db/teams","hooks_url":"https://api.github.com/repos/m3db/m3db/hooks","issue_events_url":"https://api.github.com/repos/m3db/m3db/issues/events{/number}","events_url":"https://api.github.com/repos/m3db/m3db/events","assignees_url":"https://api.github.com/repos/m3db/m3db/assignees{/user}","branches_url":"https://api.github.com/repos/m3db/m3db/branches{/branch}","tags_url":"https://api.github.com/repos/m3db/m3db/tags","blobs_url":"https://api.github.com/repos/m3db/m3db/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/m3db/m3db/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/m3db/m3db/git/refs{/sha}","trees_url":"https://api.github.com/repos/m3db/m3db/git/trees{/sha}","statuses_url":"https://api.github.com/repos/m3db/m3db/statuses/{sha}","languages_url":"https://api.github.com/repos/m3db/m3db/languages","stargazers_url":"https://api.github.com/repos/m3db/m3db/stargazers","contributors_url":"https://api.github.com/repos/m3db/m3db/contributors","subscribers_url":"https://api.github.com/repos/m3db/m3db/subscribers","subscription_url":"https://api.github.com/repos/m3db/m3db/subscription","commits_url":"https://api.github.com/repos/m3db/m3db/commits{/sha}","git_commits_url":"https://api.github.com/repos/m3db/m3db/git/commits{/sha}","comments_url":"https://api.github.com/repos/m3db/m3db/comments{/number}","issue_comment_url":"https://api.github.com/repos/m3db/m3db/issues/comments{/number}","contents_url":"https://api.github.com/repos/m3db/m3db/contents/{+path}","compare_url":"https://api.github.com/repos/m3db/m3db/compare/{base}...{head}","merges_url":"https://api.github.com/repos/m3db/m3db/merges","archive_url":"https://api.github.com/repos/m3db/m3db/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/m3db/m3db/downloads","issues_url":"https://api.github.com/repos/m3db/m3db/issues{/number}","pulls_url":"https://api.github.com/repos/m3db/m3db/pulls{/number}","milestones_url":"https://api.github.com/repos/m3db/m3db/milestones{/number}","notifications_url":"https://api.github.com/repos/m3db/m3db/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/m3db/m3db/labels{/name}","releases_url":"https://api.github.com/repos/m3db/m3db/releases{/id}","deployments_url":"https://api.github.com/repos/m3db/m3db/deployments","created_at":"2016-06-14T20:29:45Z","updated_at":"2017-07-14T05:20:41Z","pushed_at":"2017-07-28T00:35:42Z","git_url":"git://github.com/m3db/m3db.git","ssh_url":"git@github.com:m3db/m3db.git","clone_url":"https://github.com/m3db/m3db.git","svn_url":"https://github.com/m3db/m3db","homepage":"http://uber.github.io/","size":4430,"stargazers_count":110,"watchers_count":110,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":20,"forks":2,"open_issues":20,"watchers":110,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/m3db/m3db/pulls/321"},"html":{"href":"https://github.com/m3db/m3db/pull/321"},"issue":{"href":"https://api.github.com/repos/m3db/m3db/issues/321"},"comments":{"href":"https://api.github.com/repos/m3db/m3db/issues/321/comments"},"review_comments":{"href":"https://api.github.com/repos/m3db/m3db/pulls/321/comments"},"review_comment":{"href":"https://api.github.com/repos/m3db/m3db/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/m3db/m3db/pulls/321/commits"},"statuses":{"href":"https://api.github.com/repos/m3db/m3db/statuses/02e877740f616c0710ecbc759f6098e4d7af8e99"}}}}
{"actor":{"display_login":"prateek"}}