{"action":"created","comment":{"url":"https://api.github.com/repos/docker/docker/pulls/comments/96083693","pull_request_review_id":16662376,"id":96083693,"diff_hunk":"@@ -0,0 +1,176 @@\n+package cachedir\n+\n+import (\n+\t\"fmt\"\n+\t\"io/ioutil\"\n+\t\"os\"\n+\t\"path/filepath\"\n+\t\"strings\"\n+\t\"sync\"\n+\n+\t\"github.com/Sirupsen/logrus\"\n+\tdigest \"github.com/opencontainers/go-digest\"\n+)\n+\n+// Manager handles the creation and deletion of cache directories and has a\n+// very particular way of managing the cache. It was designed to be used by a\n+// caller that manages the lifetime of the directories, but delegates the\n+// mechanics of how to store and locate the directories to the Manager\n+type Manager interface {\n+\t// GetDir creates a new cache dir or returns an existing one if one already\n+\t// exists for this key from a previous download. It also keeps track of how\n+\t// many references were made to the cache dir during\n+\t// ReleaseDir or DeleteDir have to be called exactly once per GetDir call\n+\t// and GetDir shouldn't be called again until one or the other is called.\n+\tGetDir(key string) string\n+\t// ReleaseDir lets the Manager know that the dir is no longer in use but\n+\t// should be kept around in case we need it again.\n+\tReleaseDir(key string)\n+\t// DeleteDir lets the manager know that the dir is no longer used and can be\n+\t// deleted. If the delete operation fails and returns an error, the key is\n+\t// still considered released.\n+\tDeleteDir(key string) error\n+\t// CollectGarbage deletes all dirs that are not currently in use (ones that\n+\t// don't have a GetDir without a matching ReleaseDir/DeleteDir call) and\n+\t// returns the amount of space saved in bytes.\n+\tCollectGarbage() (uint64, error)\n+}\n+\n+type manager struct {\n+\tcacheRoot string\n+\tmu        sync.Mutex\n+\tinUse     map[digest.Digest]struct{}\n+}\n+\n+// NewManager creates a new cachedir manager\n+func NewManager(path string) Manager {\n+\treturn &manager{\n+\t\tcacheRoot: path,\n+\t\tinUse:     map[digest.Digest]struct{}{},\n+\t}\n+}\n+\n+func (c *manager) cachePathForKeyHash(keyHash digest.Digest) string {\n+\treturn filepath.Join(c.cacheRoot, keyHash.Algorithm().String(), keyHash.Hex())\n+}\n+\n+func dirSize(path string) (uint64, error) {\n+\tvar size uint64\n+\terr := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {\n+\t\tif info != nil && !info.IsDir() {\n+\t\t\tsize += uint64(info.Size())\n+\t\t}\n+\t\treturn err\n+\t})\n+\treturn size, err\n+}\n+\n+func (c *manager) sizeEstimateByKeyHash(keyHash digest.Digest) uint64 {\n+\tdir := c.cachePathForKeyHash(keyHash)\n+\tsize, err := dirSize(dir)\n+\tif err != nil {\n+\t\tlogrus.WithError(err).WithField(\"cachedir\", dir).Warnf(\"failed to determine size of directory\")\n+\t\treturn 0\n+\t}\n+\treturn size\n+}\n+\n+func (c *manager) cleanUpByKeyHash(keyHash digest.Digest) error {\n+\treturn os.RemoveAll(c.cachePathForKeyHash(keyHash))\n+}\n+\n+func (c *manager) GetDir(key string) string {\n+\tc.mu.Lock()\n+\tdefer c.mu.Unlock()\n+\thashedKey := digest.FromString(key)\n+\tlogrus.WithField(\"key\", key).WithField(\"hashedKey\", hashedKey).Debugf(\"cachedir manager get\")\n+\tif _, ok := c.inUse[hashedKey]; ok {\n+\t\tpanic(fmt.Errorf(\"tried to get a cachedir key (%s) twice without releasing/deleting\", key))\n+\t}\n+\tc.inUse[hashedKey] = struct{}{}\n+\n+\treturn c.cachePathForKeyHash(hashedKey)\n+}\n+\n+func (c *manager) ReleaseDir(key string) {\n+\tc.mu.Lock()\n+\tdefer c.mu.Unlock()\n+\thashedKey := digest.FromString(key)\n+\tlogrus.WithField(\"key\", key).WithField(\"hashedKey\", hashedKey).Debugf(\"cachedir manager release\")","path":"distribution/cachedir/manager.go","position":99,"original_position":99,"commit_id":"d464042ff179da8d5ad4e6783419f104ab30b714","original_commit_id":"d464042ff179da8d5ad4e6783419f104ab30b714","user":{"login":"aaronlehmann","id":10601430,"avatar_url":"https://avatars.githubusercontent.com/u/10601430?v=3","gravatar_id":"","url":"https://api.github.com/users/aaronlehmann","html_url":"https://github.com/aaronlehmann","followers_url":"https://api.github.com/users/aaronlehmann/followers","following_url":"https://api.github.com/users/aaronlehmann/following{/other_user}","gists_url":"https://api.github.com/users/aaronlehmann/gists{/gist_id}","starred_url":"https://api.github.com/users/aaronlehmann/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aaronlehmann/subscriptions","organizations_url":"https://api.github.com/users/aaronlehmann/orgs","repos_url":"https://api.github.com/users/aaronlehmann/repos","events_url":"https://api.github.com/users/aaronlehmann/events{/privacy}","received_events_url":"https://api.github.com/users/aaronlehmann/received_events","type":"User","site_admin":false},"body":"nit: `Debug` instead of `Debugf`","created_at":"2017-01-13T22:08:15Z","updated_at":"2017-01-13T22:08:15Z","html_url":"https://github.com/docker/docker/pull/28348#discussion_r96083693","pull_request_url":"https://api.github.com/repos/docker/docker/pulls/28348","_links":{"self":{"href":"https://api.github.com/repos/docker/docker/pulls/comments/96083693"},"html":{"href":"https://github.com/docker/docker/pull/28348#discussion_r96083693"},"pull_request":{"href":"https://api.github.com/repos/docker/docker/pulls/28348"}}},"pull_request":{"url":"https://api.github.com/repos/docker/docker/pulls/28348","id":93459301,"html_url":"https://github.com/docker/docker/pull/28348","diff_url":"https://github.com/docker/docker/pull/28348.diff","patch_url":"https://github.com/docker/docker/pull/28348.patch","issue_url":"https://api.github.com/repos/docker/docker/issues/28348","number":28348,"state":"open","locked":false,"title":"cache partial downloads from docker pull","user":{"login":"vikstrous","id":482425,"avatar_url":"https://avatars.githubusercontent.com/u/482425?v=3","gravatar_id":"","url":"https://api.github.com/users/vikstrous","html_url":"https://github.com/vikstrous","followers_url":"https://api.github.com/users/vikstrous/followers","following_url":"https://api.github.com/users/vikstrous/following{/other_user}","gists_url":"https://api.github.com/users/vikstrous/gists{/gist_id}","starred_url":"https://api.github.com/users/vikstrous/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vikstrous/subscriptions","organizations_url":"https://api.github.com/users/vikstrous/orgs","repos_url":"https://api.github.com/users/vikstrous/repos","events_url":"https://api.github.com/users/vikstrous/events{/privacy}","received_events_url":"https://api.github.com/users/vikstrous/received_events","type":"User","site_admin":false},"body":"I'm much more confident in this PR than the previous one because it's much smaller. It basically worked on the first try since I had all the code and already understood the architecture.\r\n\r\n**- What I did**\r\n* [x] keep partial downloads accross daemon carshes, restarts and pull cancellations\r\n* [x] use the cache dir for v1 pulls too even though they are not resumable\r\n* [x] small refactors\r\n* [x] print more progress info\r\n* [x] make progress bars start half way through if they are resuming an operation\r\n* [x] improve prune mechanism, report freed space\r\n\r\n**- How I did it**\r\nThis is my second attempt. My first attempt in #28106 turned out to be way too complex. Now instead of caching partially downloaded manifests and downloads separately, I'm managing the lifetime of the cache form the layer download manager and giving the descriptors a cache directory to use. I also separated the details of managing the cache directory into a separate package to simplify the logic a bit.\r\n\r\n**- How to verify it**\r\nLots of pulling of big images and ctrl+c-ing, pruning, etc..\r\n\r\n**- Description for the changelog**\r\nKeep partial download caches from failed pulls\r\n\r\n**- A picture of a cute animal (not mandatory but encouraged)**\r\n![](http://www.thinkcats.com/250-wide-images/cat-opening-door-handle.png)","created_at":"2016-11-13T02:09:58Z","updated_at":"2017-01-13T22:08:15Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d397ef4906ddc3e98467db2f36b574a0b3467a94","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/docker/docker/pulls/28348/commits","review_comments_url":"https://api.github.com/repos/docker/docker/pulls/28348/comments","review_comment_url":"https://api.github.com/repos/docker/docker/pulls/comments{/number}","comments_url":"https://api.github.com/repos/docker/docker/issues/28348/comments","statuses_url":"https://api.github.com/repos/docker/docker/statuses/d464042ff179da8d5ad4e6783419f104ab30b714","head":{"label":"vikstrous:26146-cache-partial-downloads-2","ref":"26146-cache-partial-downloads-2","sha":"d464042ff179da8d5ad4e6783419f104ab30b714","user":{"login":"vikstrous","id":482425,"avatar_url":"https://avatars.githubusercontent.com/u/482425?v=3","gravatar_id":"","url":"https://api.github.com/users/vikstrous","html_url":"https://github.com/vikstrous","followers_url":"https://api.github.com/users/vikstrous/followers","following_url":"https://api.github.com/users/vikstrous/following{/other_user}","gists_url":"https://api.github.com/users/vikstrous/gists{/gist_id}","starred_url":"https://api.github.com/users/vikstrous/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vikstrous/subscriptions","organizations_url":"https://api.github.com/users/vikstrous/orgs","repos_url":"https://api.github.com/users/vikstrous/repos","events_url":"https://api.github.com/users/vikstrous/events{/privacy}","received_events_url":"https://api.github.com/users/vikstrous/received_events","type":"User","site_admin":false},"repo":{"id":44000629,"name":"docker","full_name":"vikstrous/docker","owner":{"login":"vikstrous","id":482425,"avatar_url":"https://avatars.githubusercontent.com/u/482425?v=3","gravatar_id":"","url":"https://api.github.com/users/vikstrous","html_url":"https://github.com/vikstrous","followers_url":"https://api.github.com/users/vikstrous/followers","following_url":"https://api.github.com/users/vikstrous/following{/other_user}","gists_url":"https://api.github.com/users/vikstrous/gists{/gist_id}","starred_url":"https://api.github.com/users/vikstrous/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vikstrous/subscriptions","organizations_url":"https://api.github.com/users/vikstrous/orgs","repos_url":"https://api.github.com/users/vikstrous/repos","events_url":"https://api.github.com/users/vikstrous/events{/privacy}","received_events_url":"https://api.github.com/users/vikstrous/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/vikstrous/docker","description":"Docker - the open-source application container engine","fork":true,"url":"https://api.github.com/repos/vikstrous/docker","forks_url":"https://api.github.com/repos/vikstrous/docker/forks","keys_url":"https://api.github.com/repos/vikstrous/docker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/vikstrous/docker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/vikstrous/docker/teams","hooks_url":"https://api.github.com/repos/vikstrous/docker/hooks","issue_events_url":"https://api.github.com/repos/vikstrous/docker/issues/events{/number}","events_url":"https://api.github.com/repos/vikstrous/docker/events","assignees_url":"https://api.github.com/repos/vikstrous/docker/assignees{/user}","branches_url":"https://api.github.com/repos/vikstrous/docker/branches{/branch}","tags_url":"https://api.github.com/repos/vikstrous/docker/tags","blobs_url":"https://api.github.com/repos/vikstrous/docker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/vikstrous/docker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/vikstrous/docker/git/refs{/sha}","trees_url":"https://api.github.com/repos/vikstrous/docker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/vikstrous/docker/statuses/{sha}","languages_url":"https://api.github.com/repos/vikstrous/docker/languages","stargazers_url":"https://api.github.com/repos/vikstrous/docker/stargazers","contributors_url":"https://api.github.com/repos/vikstrous/docker/contributors","subscribers_url":"https://api.github.com/repos/vikstrous/docker/subscribers","subscription_url":"https://api.github.com/repos/vikstrous/docker/subscription","commits_url":"https://api.github.com/repos/vikstrous/docker/commits{/sha}","git_commits_url":"https://api.github.com/repos/vikstrous/docker/git/commits{/sha}","comments_url":"https://api.github.com/repos/vikstrous/docker/comments{/number}","issue_comment_url":"https://api.github.com/repos/vikstrous/docker/issues/comments{/number}","contents_url":"https://api.github.com/repos/vikstrous/docker/contents/{+path}","compare_url":"https://api.github.com/repos/vikstrous/docker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/vikstrous/docker/merges","archive_url":"https://api.github.com/repos/vikstrous/docker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/vikstrous/docker/downloads","issues_url":"https://api.github.com/repos/vikstrous/docker/issues{/number}","pulls_url":"https://api.github.com/repos/vikstrous/docker/pulls{/number}","milestones_url":"https://api.github.com/repos/vikstrous/docker/milestones{/number}","notifications_url":"https://api.github.com/repos/vikstrous/docker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/vikstrous/docker/labels{/name}","releases_url":"https://api.github.com/repos/vikstrous/docker/releases{/id}","deployments_url":"https://api.github.com/repos/vikstrous/docker/deployments","created_at":"2015-10-10T07:53:28Z","updated_at":"2015-10-10T07:53:40Z","pushed_at":"2017-01-12T05:23:27Z","git_url":"git://github.com/vikstrous/docker.git","ssh_url":"git@github.com:vikstrous/docker.git","clone_url":"https://github.com/vikstrous/docker.git","svn_url":"https://github.com/vikstrous/docker","homepage":"http://www.docker.com","size":117359,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"docker:master","ref":"master","sha":"d9a4bba398031b6f644985d50265df9b32ce2ae0","user":{"login":"docker","id":5429470,"avatar_url":"https://avatars.githubusercontent.com/u/5429470?v=3","gravatar_id":"","url":"https://api.github.com/users/docker","html_url":"https://github.com/docker","followers_url":"https://api.github.com/users/docker/followers","following_url":"https://api.github.com/users/docker/following{/other_user}","gists_url":"https://api.github.com/users/docker/gists{/gist_id}","starred_url":"https://api.github.com/users/docker/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/docker/subscriptions","organizations_url":"https://api.github.com/users/docker/orgs","repos_url":"https://api.github.com/users/docker/repos","events_url":"https://api.github.com/users/docker/events{/privacy}","received_events_url":"https://api.github.com/users/docker/received_events","type":"Organization","site_admin":false},"repo":{"id":7691631,"name":"docker","full_name":"docker/docker","owner":{"login":"docker","id":5429470,"avatar_url":"https://avatars.githubusercontent.com/u/5429470?v=3","gravatar_id":"","url":"https://api.github.com/users/docker","html_url":"https://github.com/docker","followers_url":"https://api.github.com/users/docker/followers","following_url":"https://api.github.com/users/docker/following{/other_user}","gists_url":"https://api.github.com/users/docker/gists{/gist_id}","starred_url":"https://api.github.com/users/docker/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/docker/subscriptions","organizations_url":"https://api.github.com/users/docker/orgs","repos_url":"https://api.github.com/users/docker/repos","events_url":"https://api.github.com/users/docker/events{/privacy}","received_events_url":"https://api.github.com/users/docker/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/docker/docker","description":"Docker - the open-source application container engine","fork":false,"url":"https://api.github.com/repos/docker/docker","forks_url":"https://api.github.com/repos/docker/docker/forks","keys_url":"https://api.github.com/repos/docker/docker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/docker/docker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/docker/docker/teams","hooks_url":"https://api.github.com/repos/docker/docker/hooks","issue_events_url":"https://api.github.com/repos/docker/docker/issues/events{/number}","events_url":"https://api.github.com/repos/docker/docker/events","assignees_url":"https://api.github.com/repos/docker/docker/assignees{/user}","branches_url":"https://api.github.com/repos/docker/docker/branches{/branch}","tags_url":"https://api.github.com/repos/docker/docker/tags","blobs_url":"https://api.github.com/repos/docker/docker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/docker/docker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/docker/docker/git/refs{/sha}","trees_url":"https://api.github.com/repos/docker/docker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/docker/docker/statuses/{sha}","languages_url":"https://api.github.com/repos/docker/docker/languages","stargazers_url":"https://api.github.com/repos/docker/docker/stargazers","contributors_url":"https://api.github.com/repos/docker/docker/contributors","subscribers_url":"https://api.github.com/repos/docker/docker/subscribers","subscription_url":"https://api.github.com/repos/docker/docker/subscription","commits_url":"https://api.github.com/repos/docker/docker/commits{/sha}","git_commits_url":"https://api.github.com/repos/docker/docker/git/commits{/sha}","comments_url":"https://api.github.com/repos/docker/docker/comments{/number}","issue_comment_url":"https://api.github.com/repos/docker/docker/issues/comments{/number}","contents_url":"https://api.github.com/repos/docker/docker/contents/{+path}","compare_url":"https://api.github.com/repos/docker/docker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/docker/docker/merges","archive_url":"https://api.github.com/repos/docker/docker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/docker/docker/downloads","issues_url":"https://api.github.com/repos/docker/docker/issues{/number}","pulls_url":"https://api.github.com/repos/docker/docker/pulls{/number}","milestones_url":"https://api.github.com/repos/docker/docker/milestones{/number}","notifications_url":"https://api.github.com/repos/docker/docker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/docker/docker/labels{/name}","releases_url":"https://api.github.com/repos/docker/docker/releases{/id}","deployments_url":"https://api.github.com/repos/docker/docker/deployments","created_at":"2013-01-18T18:10:57Z","updated_at":"2017-01-13T21:54:54Z","pushed_at":"2017-01-13T21:59:13Z","git_url":"git://github.com/docker/docker.git","ssh_url":"git@github.com:docker/docker.git","clone_url":"https://github.com/docker/docker.git","svn_url":"https://github.com/docker/docker","homepage":"http://www.docker.com","size":120777,"stargazers_count":38689,"watchers_count":38689,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11532,"mirror_url":null,"open_issues_count":2135,"forks":11532,"open_issues":2135,"watchers":38689,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/docker/docker/pulls/28348"},"html":{"href":"https://github.com/docker/docker/pull/28348"},"issue":{"href":"https://api.github.com/repos/docker/docker/issues/28348"},"comments":{"href":"https://api.github.com/repos/docker/docker/issues/28348/comments"},"review_comments":{"href":"https://api.github.com/repos/docker/docker/pulls/28348/comments"},"review_comment":{"href":"https://api.github.com/repos/docker/docker/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/docker/docker/pulls/28348/commits"},"statuses":{"href":"https://api.github.com/repos/docker/docker/statuses/d464042ff179da8d5ad4e6783419f104ab30b714"}}}}
{"actor":{"display_login":"aaronlehmann"}}