{"action":"created","comment":{"url":"https://api.github.com/repos/google/trillian/pulls/comments/141668007","pull_request_review_id":65918226,"id":141668007,"diff_hunk":"@@ -0,0 +1,121 @@\n+// Copyright 2017 Google Inc. All Rights Reserved.\n+//\n+// Licensed under the Apache License, Version 2.0 (the \"License\");\n+// you may not use this file except in compliance with the License.\n+// You may obtain a copy of the License at\n+//\n+//     http://www.apache.org/licenses/LICENSE-2.0\n+//\n+// Unless required by applicable law or agreed to in writing, software\n+// distributed under the License is distributed on an \"AS IS\" BASIS,\n+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+// See the License for the specific language governing permissions and\n+// limitations under the License.\n+\n+package admin\n+\n+import (\n+\t\"context\"\n+\t\"math/rand\"\n+\t\"time\"\n+\n+\t\"github.com/golang/glog\"\n+\t\"github.com/golang/protobuf/ptypes\"\n+\t\"github.com/google/trillian\"\n+\t\"github.com/google/trillian/storage\"\n+)\n+\n+var (\n+\ttimeNow   = time.Now\n+\ttimeSleep = time.Sleep\n+)\n+\n+// DeletedTreeGC garbage collects deleted trees.\n+//\n+// Tree deletion goes through two separate stages:\n+// * Soft deletion, which \"flips a bit\" (see Tree.Deleted) but otherwise leaves the tree unaltered\n+// * Hard deletion, which effectively removes all tree data\n+//\n+// DeletedTreeGC performs the transition from soft to hard deletion. Trees that have been deleted\n+// for at least DeletedThreshold are eligible for garbage collection.\n+type DeletedTreeGC struct {\n+\t// Admin is the storage.AdminStorage interface.\n+\tAdmin storage.AdminStorage\n+\n+\t// DeleteThreshold defines the minimum time a tree has to remain in the soft-deleted state\n+\t// before its eligible for garbage collection.\n+\tDeleteThreshold time.Duration\n+\n+\t// MinRunInterval defines how frequently sweeps for deleted trees are performed.\n+\t// Actual runs happen randomly between [minInterval,2*minInterval).\n+\tMinRunInterval time.Duration\n+}\n+\n+// Run starts the tree garbage collection process. It never returns.\n+func (gc *DeletedTreeGC) Run(ctx context.Context) {\n+\tfor true {\n+\t\tgc.RunOnce(ctx)\n+\n+\t\td := gc.MinRunInterval + time.Duration(rand.Int63n(gc.MinRunInterval.Nanoseconds()))\n+\t\ttimeSleep(d)\n+\t}\n+}\n+\n+// RunOnce performs a single tree garbage collection sweep.\n+// RunOnce never errors, instead it attempts to delete as many eligible trees as possible. Failures\n+// are simply logged.\n+func (gc *DeletedTreeGC) RunOnce(ctx context.Context) {\n+\tnow := timeNow()\n+\n+\t// List and delete trees in separate transactions. Hard-deletes may cascade to a lot of data, so\n+\t// each delete should be in its own transaction as well.\n+\t// It's OK to list and delete separately because HardDelete does its own state checking, plus\n+\t// deleted trees are unlikely to change, specially those deleted for a while.\n+\ttx, err := gc.Admin.Snapshot(ctx)\n+\tif err != nil {\n+\t\tglog.Errorf(\"DeletedTreeGC.RunOnce: error creating snapshot: %v\", err)\n+\t\treturn\n+\t}\n+\tdefer tx.Close()\n+\ttrees, err := tx.ListTrees(ctx, true /* includeDeleted */)\n+\tif err != nil {\n+\t\tglog.Errorf(\"DeletedTreeGC.RunOnce: error listing trees: %v\", err)","path":"server/admin/tree_gc.go","position":82,"original_position":82,"commit_id":"8098b42ad16872823bec6af17f147efe4e268471","original_commit_id":"8098b42ad16872823bec6af17f147efe4e268471","user":{"login":"AlCutter","id":7648032,"avatar_url":"https://avatars0.githubusercontent.com/u/7648032?v=4","gravatar_id":"","url":"https://api.github.com/users/AlCutter","html_url":"https://github.com/AlCutter","followers_url":"https://api.github.com/users/AlCutter/followers","following_url":"https://api.github.com/users/AlCutter/following{/other_user}","gists_url":"https://api.github.com/users/AlCutter/gists{/gist_id}","starred_url":"https://api.github.com/users/AlCutter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AlCutter/subscriptions","organizations_url":"https://api.github.com/users/AlCutter/orgs","repos_url":"https://api.github.com/users/AlCutter/repos","events_url":"https://api.github.com/users/AlCutter/events{/privacy}","received_events_url":"https://api.github.com/users/AlCutter/received_events","type":"User","site_admin":false},"body":"Should we increment a counter on these fails so that the operator can be alerted?","created_at":"2017-09-28T16:23:59Z","updated_at":"2017-09-28T16:25:56Z","html_url":"https://github.com/google/trillian/pull/844#discussion_r141668007","pull_request_url":"https://api.github.com/repos/google/trillian/pulls/844","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/google/trillian/pulls/comments/141668007"},"html":{"href":"https://github.com/google/trillian/pull/844#discussion_r141668007"},"pull_request":{"href":"https://api.github.com/repos/google/trillian/pulls/844"}}},"pull_request":{"url":"https://api.github.com/repos/google/trillian/pulls/844","id":143370310,"html_url":"https://github.com/google/trillian/pull/844","diff_url":"https://github.com/google/trillian/pull/844.diff","patch_url":"https://github.com/google/trillian/pull/844.patch","issue_url":"https://api.github.com/repos/google/trillian/issues/844","number":844,"state":"open","locked":false,"title":"Tree garbage collection","user":{"login":"codingllama","id":12500300,"avatar_url":"https://avatars3.githubusercontent.com/u/12500300?v=4","gravatar_id":"","url":"https://api.github.com/users/codingllama","html_url":"https://github.com/codingllama","followers_url":"https://api.github.com/users/codingllama/followers","following_url":"https://api.github.com/users/codingllama/following{/other_user}","gists_url":"https://api.github.com/users/codingllama/gists{/gist_id}","starred_url":"https://api.github.com/users/codingllama/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codingllama/subscriptions","organizations_url":"https://api.github.com/users/codingllama/orgs","repos_url":"https://api.github.com/users/codingllama/repos","events_url":"https://api.github.com/users/codingllama/events{/privacy}","received_events_url":"https://api.github.com/users/codingllama/received_events","type":"User","site_admin":false},"body":"Create a background task that periodically scans for and hard-deletes trees that have been soft-deleted longer than the specified period.","created_at":"2017-09-27T11:47:04Z","updated_at":"2017-09-28T16:25:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"da3727656baf8038dc0cc8dd02b712eceb897286","assignee":{"login":"AlCutter","id":7648032,"avatar_url":"https://avatars0.githubusercontent.com/u/7648032?v=4","gravatar_id":"","url":"https://api.github.com/users/AlCutter","html_url":"https://github.com/AlCutter","followers_url":"https://api.github.com/users/AlCutter/followers","following_url":"https://api.github.com/users/AlCutter/following{/other_user}","gists_url":"https://api.github.com/users/AlCutter/gists{/gist_id}","starred_url":"https://api.github.com/users/AlCutter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AlCutter/subscriptions","organizations_url":"https://api.github.com/users/AlCutter/orgs","repos_url":"https://api.github.com/users/AlCutter/repos","events_url":"https://api.github.com/users/AlCutter/events{/privacy}","received_events_url":"https://api.github.com/users/AlCutter/received_events","type":"User","site_admin":false},"assignees":[{"login":"AlCutter","id":7648032,"avatar_url":"https://avatars0.githubusercontent.com/u/7648032?v=4","gravatar_id":"","url":"https://api.github.com/users/AlCutter","html_url":"https://github.com/AlCutter","followers_url":"https://api.github.com/users/AlCutter/followers","following_url":"https://api.github.com/users/AlCutter/following{/other_user}","gists_url":"https://api.github.com/users/AlCutter/gists{/gist_id}","starred_url":"https://api.github.com/users/AlCutter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AlCutter/subscriptions","organizations_url":"https://api.github.com/users/AlCutter/orgs","repos_url":"https://api.github.com/users/AlCutter/repos","events_url":"https://api.github.com/users/AlCutter/events{/privacy}","received_events_url":"https://api.github.com/users/AlCutter/received_events","type":"User","site_admin":false},{"login":"codingllama","id":12500300,"avatar_url":"https://avatars3.githubusercontent.com/u/12500300?v=4","gravatar_id":"","url":"https://api.github.com/users/codingllama","html_url":"https://github.com/codingllama","followers_url":"https://api.github.com/users/codingllama/followers","following_url":"https://api.github.com/users/codingllama/following{/other_user}","gists_url":"https://api.github.com/users/codingllama/gists{/gist_id}","starred_url":"https://api.github.com/users/codingllama/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codingllama/subscriptions","organizations_url":"https://api.github.com/users/codingllama/orgs","repos_url":"https://api.github.com/users/codingllama/repos","events_url":"https://api.github.com/users/codingllama/events{/privacy}","received_events_url":"https://api.github.com/users/codingllama/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/google/trillian/pulls/844/commits","review_comments_url":"https://api.github.com/repos/google/trillian/pulls/844/comments","review_comment_url":"https://api.github.com/repos/google/trillian/pulls/comments{/number}","comments_url":"https://api.github.com/repos/google/trillian/issues/844/comments","statuses_url":"https://api.github.com/repos/google/trillian/statuses/8098b42ad16872823bec6af17f147efe4e268471","head":{"label":"codingllama:tree-gc","ref":"tree-gc","sha":"8098b42ad16872823bec6af17f147efe4e268471","user":{"login":"codingllama","id":12500300,"avatar_url":"https://avatars3.githubusercontent.com/u/12500300?v=4","gravatar_id":"","url":"https://api.github.com/users/codingllama","html_url":"https://github.com/codingllama","followers_url":"https://api.github.com/users/codingllama/followers","following_url":"https://api.github.com/users/codingllama/following{/other_user}","gists_url":"https://api.github.com/users/codingllama/gists{/gist_id}","starred_url":"https://api.github.com/users/codingllama/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codingllama/subscriptions","organizations_url":"https://api.github.com/users/codingllama/orgs","repos_url":"https://api.github.com/users/codingllama/repos","events_url":"https://api.github.com/users/codingllama/events{/privacy}","received_events_url":"https://api.github.com/users/codingllama/received_events","type":"User","site_admin":false},"repo":{"id":75955988,"name":"trillian","full_name":"codingllama/trillian","owner":{"login":"codingllama","id":12500300,"avatar_url":"https://avatars3.githubusercontent.com/u/12500300?v=4","gravatar_id":"","url":"https://api.github.com/users/codingllama","html_url":"https://github.com/codingllama","followers_url":"https://api.github.com/users/codingllama/followers","following_url":"https://api.github.com/users/codingllama/following{/other_user}","gists_url":"https://api.github.com/users/codingllama/gists{/gist_id}","starred_url":"https://api.github.com/users/codingllama/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codingllama/subscriptions","organizations_url":"https://api.github.com/users/codingllama/orgs","repos_url":"https://api.github.com/users/codingllama/repos","events_url":"https://api.github.com/users/codingllama/events{/privacy}","received_events_url":"https://api.github.com/users/codingllama/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/codingllama/trillian","description":null,"fork":true,"url":"https://api.github.com/repos/codingllama/trillian","forks_url":"https://api.github.com/repos/codingllama/trillian/forks","keys_url":"https://api.github.com/repos/codingllama/trillian/keys{/key_id}","collaborators_url":"https://api.github.com/repos/codingllama/trillian/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/codingllama/trillian/teams","hooks_url":"https://api.github.com/repos/codingllama/trillian/hooks","issue_events_url":"https://api.github.com/repos/codingllama/trillian/issues/events{/number}","events_url":"https://api.github.com/repos/codingllama/trillian/events","assignees_url":"https://api.github.com/repos/codingllama/trillian/assignees{/user}","branches_url":"https://api.github.com/repos/codingllama/trillian/branches{/branch}","tags_url":"https://api.github.com/repos/codingllama/trillian/tags","blobs_url":"https://api.github.com/repos/codingllama/trillian/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/codingllama/trillian/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/codingllama/trillian/git/refs{/sha}","trees_url":"https://api.github.com/repos/codingllama/trillian/git/trees{/sha}","statuses_url":"https://api.github.com/repos/codingllama/trillian/statuses/{sha}","languages_url":"https://api.github.com/repos/codingllama/trillian/languages","stargazers_url":"https://api.github.com/repos/codingllama/trillian/stargazers","contributors_url":"https://api.github.com/repos/codingllama/trillian/contributors","subscribers_url":"https://api.github.com/repos/codingllama/trillian/subscribers","subscription_url":"https://api.github.com/repos/codingllama/trillian/subscription","commits_url":"https://api.github.com/repos/codingllama/trillian/commits{/sha}","git_commits_url":"https://api.github.com/repos/codingllama/trillian/git/commits{/sha}","comments_url":"https://api.github.com/repos/codingllama/trillian/comments{/number}","issue_comment_url":"https://api.github.com/repos/codingllama/trillian/issues/comments{/number}","contents_url":"https://api.github.com/repos/codingllama/trillian/contents/{+path}","compare_url":"https://api.github.com/repos/codingllama/trillian/compare/{base}...{head}","merges_url":"https://api.github.com/repos/codingllama/trillian/merges","archive_url":"https://api.github.com/repos/codingllama/trillian/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/codingllama/trillian/downloads","issues_url":"https://api.github.com/repos/codingllama/trillian/issues{/number}","pulls_url":"https://api.github.com/repos/codingllama/trillian/pulls{/number}","milestones_url":"https://api.github.com/repos/codingllama/trillian/milestones{/number}","notifications_url":"https://api.github.com/repos/codingllama/trillian/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/codingllama/trillian/labels{/name}","releases_url":"https://api.github.com/repos/codingllama/trillian/releases{/id}","deployments_url":"https://api.github.com/repos/codingllama/trillian/deployments","created_at":"2016-12-08T16:36:46Z","updated_at":"2016-12-08T16:36:48Z","pushed_at":"2017-09-28T11:25:43Z","git_url":"git://github.com/codingllama/trillian.git","ssh_url":"git@github.com:codingllama/trillian.git","clone_url":"https://github.com/codingllama/trillian.git","svn_url":"https://github.com/codingllama/trillian","homepage":null,"size":7575,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_projects":true,"has_downloads":false,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"google:master","ref":"master","sha":"c96adf0250590b8db091b3bcd14476333fce2ab1","user":{"login":"google","id":1342004,"avatar_url":"https://avatars1.githubusercontent.com/u/1342004?v=4","gravatar_id":"","url":"https://api.github.com/users/google","html_url":"https://github.com/google","followers_url":"https://api.github.com/users/google/followers","following_url":"https://api.github.com/users/google/following{/other_user}","gists_url":"https://api.github.com/users/google/gists{/gist_id}","starred_url":"https://api.github.com/users/google/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/google/subscriptions","organizations_url":"https://api.github.com/users/google/orgs","repos_url":"https://api.github.com/users/google/repos","events_url":"https://api.github.com/users/google/events{/privacy}","received_events_url":"https://api.github.com/users/google/received_events","type":"Organization","site_admin":false},"repo":{"id":58654489,"name":"trillian","full_name":"google/trillian","owner":{"login":"google","id":1342004,"avatar_url":"https://avatars1.githubusercontent.com/u/1342004?v=4","gravatar_id":"","url":"https://api.github.com/users/google","html_url":"https://github.com/google","followers_url":"https://api.github.com/users/google/followers","following_url":"https://api.github.com/users/google/following{/other_user}","gists_url":"https://api.github.com/users/google/gists{/gist_id}","starred_url":"https://api.github.com/users/google/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/google/subscriptions","organizations_url":"https://api.github.com/users/google/orgs","repos_url":"https://api.github.com/users/google/repos","events_url":"https://api.github.com/users/google/events{/privacy}","received_events_url":"https://api.github.com/users/google/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/google/trillian","description":"Trillian implements a Merkle tree whose contents are served from a data storage layer, to allow scalability to extremely large trees.","fork":false,"url":"https://api.github.com/repos/google/trillian","forks_url":"https://api.github.com/repos/google/trillian/forks","keys_url":"https://api.github.com/repos/google/trillian/keys{/key_id}","collaborators_url":"https://api.github.com/repos/google/trillian/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/google/trillian/teams","hooks_url":"https://api.github.com/repos/google/trillian/hooks","issue_events_url":"https://api.github.com/repos/google/trillian/issues/events{/number}","events_url":"https://api.github.com/repos/google/trillian/events","assignees_url":"https://api.github.com/repos/google/trillian/assignees{/user}","branches_url":"https://api.github.com/repos/google/trillian/branches{/branch}","tags_url":"https://api.github.com/repos/google/trillian/tags","blobs_url":"https://api.github.com/repos/google/trillian/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/google/trillian/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/google/trillian/git/refs{/sha}","trees_url":"https://api.github.com/repos/google/trillian/git/trees{/sha}","statuses_url":"https://api.github.com/repos/google/trillian/statuses/{sha}","languages_url":"https://api.github.com/repos/google/trillian/languages","stargazers_url":"https://api.github.com/repos/google/trillian/stargazers","contributors_url":"https://api.github.com/repos/google/trillian/contributors","subscribers_url":"https://api.github.com/repos/google/trillian/subscribers","subscription_url":"https://api.github.com/repos/google/trillian/subscription","commits_url":"https://api.github.com/repos/google/trillian/commits{/sha}","git_commits_url":"https://api.github.com/repos/google/trillian/git/commits{/sha}","comments_url":"https://api.github.com/repos/google/trillian/comments{/number}","issue_comment_url":"https://api.github.com/repos/google/trillian/issues/comments{/number}","contents_url":"https://api.github.com/repos/google/trillian/contents/{+path}","compare_url":"https://api.github.com/repos/google/trillian/compare/{base}...{head}","merges_url":"https://api.github.com/repos/google/trillian/merges","archive_url":"https://api.github.com/repos/google/trillian/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/google/trillian/downloads","issues_url":"https://api.github.com/repos/google/trillian/issues{/number}","pulls_url":"https://api.github.com/repos/google/trillian/pulls{/number}","milestones_url":"https://api.github.com/repos/google/trillian/milestones{/number}","notifications_url":"https://api.github.com/repos/google/trillian/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/google/trillian/labels{/name}","releases_url":"https://api.github.com/repos/google/trillian/releases{/id}","deployments_url":"https://api.github.com/repos/google/trillian/deployments","created_at":"2016-05-12T15:37:35Z","updated_at":"2017-09-28T14:25:39Z","pushed_at":"2017-09-28T15:49:50Z","git_url":"git://github.com/google/trillian.git","ssh_url":"git@github.com:google/trillian.git","clone_url":"https://github.com/google/trillian.git","svn_url":"https://github.com/google/trillian","homepage":"","size":7567,"stargazers_count":632,"watchers_count":632,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":false,"has_wiki":false,"has_pages":false,"forks_count":90,"mirror_url":null,"open_issues_count":41,"forks":90,"open_issues":41,"watchers":632,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/google/trillian/pulls/844"},"html":{"href":"https://github.com/google/trillian/pull/844"},"issue":{"href":"https://api.github.com/repos/google/trillian/issues/844"},"comments":{"href":"https://api.github.com/repos/google/trillian/issues/844/comments"},"review_comments":{"href":"https://api.github.com/repos/google/trillian/pulls/844/comments"},"review_comment":{"href":"https://api.github.com/repos/google/trillian/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/google/trillian/pulls/844/commits"},"statuses":{"href":"https://api.github.com/repos/google/trillian/statuses/8098b42ad16872823bec6af17f147efe4e268471"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"AlCutter"}}