{"action":"created","comment":{"url":"https://api.github.com/repos/attic-labs/noms/pulls/comments/137148319","pull_request_review_id":60777082,"id":137148319,"diff_hunk":"@@ -5,49 +5,136 @@\n package main\n \n import (\n-\t\"fmt\"\n+\t\"context\"\n+\t\"encoding/base64\"\n \t\"log\"\n \t\"os\"\n \t\"time\"\n \n \t\"github.com/attic-labs/noms/go/d\"\n \t\"github.com/attic-labs/noms/go/datas\"\n+\t\"github.com/attic-labs/noms/go/hash\"\n \t\"github.com/attic-labs/noms/go/ipfs\"\n+\t\"github.com/attic-labs/noms/go/merge\"\n \t\"github.com/attic-labs/noms/go/spec\"\n+\t\"github.com/attic-labs/noms/go/types\"\n \t\"github.com/attic-labs/noms/samples/go/ipfs-chat/dbg\"\n+\t\"github.com/ipfs/go-ipfs/core\"\n )\n \n-func runDaemon(topic string, interval time.Duration, networkDS, localDS string, netNodeIdx, localNodeIdx int) {\n+func runDaemon(topic string, interval time.Duration, ipfsSpec string, nodeIdx int) {\n \tdbg.SetLogger(log.New(os.Stdout, \"\", 0))\n \n-\tipfs.NodeIndex = netNodeIdx\n-\tsourceSp, err := spec.ForDataset(networkDS)\n+\tsp, err := spec.ForDataset(ipfsSpec)\n \td.CheckErrorNoUsage(err)\n-\tsource := sourceSp.GetDataset()\n-\tsource, err = InitDatabase(source)\n-\td.PanicIfError(err)\n+\tds := sp.GetDataset()\n \n-\tipfs.NodeIndex = localNodeIdx\n-\tdestSp, err := spec.ForDataset(localDS)\n-\td.CheckErrorNoUsage(err)\n-\tdest := destSp.GetDataset()\n-\tdest, err = InitDatabase(dest)\n-\td.PanicIfError(err)\n+\tnode, cs := reconfigureIPFSChunkStore(sp, topic, nodeIdx)\n+\tsourceDB := datas.NewDatabase(cs)\n+\tsourceDS := sourceDB.GetDataset(ds.ID())\n \n-\tipfs.NodeIndex = -1\n+\tcs = ipfs.NewChunkStorePrimitive(sp.DatabaseName, true, node)\n+\tdestDB := datas.NewDatabase(cs)\n+\tdestDS := destDB.GetDataset(ds.ID())\n+\tdestDS, err = InitDatabase(destDS)\n+\td.PanicIfError(err)\n \n-\tfmt.Printf(\"Replicating %s to %s...\\n\", sourceSp.String(), destSp.String())\n+\tdbg.Debug(\"Storing locally to:\", sp.String())\n \n-\tsub, err := ipfs.CurrentNode.Floodsub.Subscribe(topic)\n-\td.PanicIfError(err)\n-\tgo Replicate(sub, source, dest, func(ds datas.Dataset) {\n-\t\tdest = ds\n+\tgo replicate(node, topic, sourceDS, destDS, func(ds1 datas.Dataset) {\n+\t\tdestDS = ds1\n \t})\n \n \tfor {\n-\t\ts := dest.HeadRef().TargetHash().String()\n-\t\tfmt.Println(\"publishing: \" + s)\n-\t\tPublish(sub, topic, s)\n+\t\tPublish(node, topic, destDS.HeadRef().TargetHash())\n \t\ttime.Sleep(interval)\n \t}\n }\n+\n+// replicate continually listens for commit hashes published by ipfs-chat nodes,\n+// ensures that all nodes are replicated locally, and merges new data into it's\n+// dataset when necessary.\n+func replicate(node *core.IpfsNode, topic string, sourceDS, destDS datas.Dataset, didChange func(ds datas.Dataset)) {\n+\tsub, err := node.Floodsub.Subscribe(topic)\n+\td.Chk.NoError(err)\n+\n+\tvar lastHash hash.Hash\n+\tfor {\n+\t\tdbg.Debug(\"looking for msgs\")\n+\t\tmsg, err := sub.Next(context.Background())\n+\t\td.PanicIfError(err)\n+\t\tmsgHash := hash.Parse(string(msg.Data))\n+\t\tdbg.Debug(\"got msg, msgHash: %s, lastHash: %s\", msgHash.String(), lastHash.String())\n+\t\tif lastHash == msgHash {\n+\t\t\tcontinue\n+\t\t}\n+\t\tlastHash = msgHash\n+\n+\t\tdbg.Debug(\"got update: %s from %s\", msgHash, base64.StdEncoding.EncodeToString(msg.From))\n+\t\tdestDB := destDS.Database()\n+\t\tdestDB.Rebase()\n+\t\tdestDS = destDB.GetDataset(destDS.ID())\n+\t\td.PanicIfFalse(destDS.HasHead())\n+\n+\t\tdbg.Debug(\"syncing commits\")\n+\t\tpullCommits(msgHash, sourceDS.Database(), destDB)\n+\n+\t\tif msgHash == destDS.HeadRef().TargetHash() {\n+\t\t\tdbg.Debug(\"received hash same as current head, nothing to do\")\n+\t\t\tcontinue\n+\t\t}\n+\t\tsourceCommit := destDB.ReadValue(msgHash)\n+\t\tsourceRef := types.NewRef(sourceCommit)\n+\t\ta, ok := datas.FindCommonAncestor(sourceRef, destDS.HeadRef(), destDB)\n+\t\tif !ok {\n+\t\t\tdbg.Debug(\"no common ancestor, cannot merge update!\")\n+\t\t\tcontinue\n+\t\t}\n+\t\tif a.Equals(sourceRef) {\n+\t\t\tdbg.Debug(\"source commit was ancestor, nothing to do\")\n+\t\t\tcontinue\n+\t\t}\n+\t\tif a.Equals(destDS.HeadRef()) {\n+\t\t\tdbg.Debug(\"fast-forward to source commit\")\n+\t\t\tdestDS, err = destDB.SetHead(destDS, sourceRef)\n+\t\t\tdidChange(destDS)\n+\t\t\tcontinue\n+\t\t}\n+\n+\t\tleft := destDS.HeadValue()\n+\t\tright := sourceCommit.(types.Struct).Get(\"value\")\n+\t\tparent := a.TargetValue(destDB).(types.Struct).Get(\"value\")\n+\n+\t\tmerged, err := merge.ThreeWay(left, right, parent, destDB, nil, nil)\n+\t\tif err != nil {\n+\t\t\tdbg.Debug(\"could not merge received data: \" + err.Error())\n+\t\t\tcontinue\n+\t\t}\n+\n+\t\tdestDS, err = destDB.SetHead(destDS, destDB.WriteValue(datas.NewCommit(merged, types.NewSet(destDB, destDS.HeadRef(), sourceRef), types.EmptyStruct)))\n+\t\tif err != nil {\n+\t\t\tdbg.Debug(\"call failed to SetHead on destDB, err: %s\", err)\n+\t\t}\n+\t\tdidChange(destDS)\n+\t}\n+}\n+\n+func pullCommits(h hash.Hash, sourceDB, destDB datas.Database) {","path":"samples/go/ipfs-chat/daemon.go","position":null,"original_position":140,"commit_id":"45c9c23b840e9b24f2436a445d5c5701a2316165","original_commit_id":"1b307c220bcc851a5458013b739222c3d6442597","user":{"login":"willhite","id":487362,"avatar_url":"https://avatars0.githubusercontent.com/u/487362?v=4","gravatar_id":"","url":"https://api.github.com/users/willhite","html_url":"https://github.com/willhite","followers_url":"https://api.github.com/users/willhite/followers","following_url":"https://api.github.com/users/willhite/following{/other_user}","gists_url":"https://api.github.com/users/willhite/gists{/gist_id}","starred_url":"https://api.github.com/users/willhite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/willhite/subscriptions","organizations_url":"https://api.github.com/users/willhite/orgs","repos_url":"https://api.github.com/users/willhite/repos","events_url":"https://api.github.com/users/willhite/events{/privacy}","received_events_url":"https://api.github.com/users/willhite/received_events","type":"User","site_admin":false},"body":"It hasn't been tested thoroughly and still seems flakey. A work in progress.","created_at":"2017-09-06T00:38:29Z","updated_at":"2017-09-06T00:42:32Z","html_url":"https://github.com/attic-labs/noms/pull/3666#discussion_r137148319","pull_request_url":"https://api.github.com/repos/attic-labs/noms/pulls/3666","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/attic-labs/noms/pulls/comments/137148319"},"html":{"href":"https://github.com/attic-labs/noms/pull/3666#discussion_r137148319"},"pull_request":{"href":"https://api.github.com/repos/attic-labs/noms/pulls/3666"}},"in_reply_to_id":136874463},"pull_request":{"url":"https://api.github.com/repos/attic-labs/noms/pulls/3666","id":139198569,"html_url":"https://github.com/attic-labs/noms/pull/3666","diff_url":"https://github.com/attic-labs/noms/pull/3666.diff","patch_url":"https://github.com/attic-labs/noms/pull/3666.patch","issue_url":"https://api.github.com/repos/attic-labs/noms/issues/3666","number":3666,"state":"open","locked":false,"title":"Modifications to ipfs-chat can ipfs chunkstore","user":{"login":"willhite","id":487362,"avatar_url":"https://avatars0.githubusercontent.com/u/487362?v=4","gravatar_id":"","url":"https://api.github.com/users/willhite","html_url":"https://github.com/willhite","followers_url":"https://api.github.com/users/willhite/followers","following_url":"https://api.github.com/users/willhite/following{/other_user}","gists_url":"https://api.github.com/users/willhite/gists{/gist_id}","starred_url":"https://api.github.com/users/willhite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/willhite/subscriptions","organizations_url":"https://api.github.com/users/willhite/orgs","repos_url":"https://api.github.com/users/willhite/repos","events_url":"https://api.github.com/users/willhite/events{/privacy}","received_events_url":"https://api.github.com/users/willhite/received_events","type":"User","site_admin":false},"body":" * Change ipfs paths to include directory where ipfs repo is stored.\r\n * Rework ipfs-chat to create ipfs chunkstores manually rather than\r\n   relying on Spec.ForDataset. This enables creating two chunkstores\r\n   (one local and one network) using the same IpfsNode (ipfs repo).\r\n * Create separate replicate function for daemon and mergeMessage\r\n   function for client to experiment with slightly different behaviors\r\n   for each.","created_at":"2017-09-04T18:20:57Z","updated_at":"2017-09-06T00:42:33Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7dd7ae30faea00112395880c77a85a7c25ded6b5","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/attic-labs/noms/pulls/3666/commits","review_comments_url":"https://api.github.com/repos/attic-labs/noms/pulls/3666/comments","review_comment_url":"https://api.github.com/repos/attic-labs/noms/pulls/comments{/number}","comments_url":"https://api.github.com/repos/attic-labs/noms/issues/3666/comments","statuses_url":"https://api.github.com/repos/attic-labs/noms/statuses/45c9c23b840e9b24f2436a445d5c5701a2316165","head":{"label":"willhite:chat-mods","ref":"chat-mods","sha":"45c9c23b840e9b24f2436a445d5c5701a2316165","user":{"login":"willhite","id":487362,"avatar_url":"https://avatars0.githubusercontent.com/u/487362?v=4","gravatar_id":"","url":"https://api.github.com/users/willhite","html_url":"https://github.com/willhite","followers_url":"https://api.github.com/users/willhite/followers","following_url":"https://api.github.com/users/willhite/following{/other_user}","gists_url":"https://api.github.com/users/willhite/gists{/gist_id}","starred_url":"https://api.github.com/users/willhite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/willhite/subscriptions","organizations_url":"https://api.github.com/users/willhite/orgs","repos_url":"https://api.github.com/users/willhite/repos","events_url":"https://api.github.com/users/willhite/events{/privacy}","received_events_url":"https://api.github.com/users/willhite/received_events","type":"User","site_admin":false},"repo":{"id":64777393,"name":"noms","full_name":"willhite/noms","owner":{"login":"willhite","id":487362,"avatar_url":"https://avatars0.githubusercontent.com/u/487362?v=4","gravatar_id":"","url":"https://api.github.com/users/willhite","html_url":"https://github.com/willhite","followers_url":"https://api.github.com/users/willhite/followers","following_url":"https://api.github.com/users/willhite/following{/other_user}","gists_url":"https://api.github.com/users/willhite/gists{/gist_id}","starred_url":"https://api.github.com/users/willhite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/willhite/subscriptions","organizations_url":"https://api.github.com/users/willhite/orgs","repos_url":"https://api.github.com/users/willhite/repos","events_url":"https://api.github.com/users/willhite/events{/privacy}","received_events_url":"https://api.github.com/users/willhite/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/willhite/noms","description":"The versioned, forkable, syncable database","fork":true,"url":"https://api.github.com/repos/willhite/noms","forks_url":"https://api.github.com/repos/willhite/noms/forks","keys_url":"https://api.github.com/repos/willhite/noms/keys{/key_id}","collaborators_url":"https://api.github.com/repos/willhite/noms/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/willhite/noms/teams","hooks_url":"https://api.github.com/repos/willhite/noms/hooks","issue_events_url":"https://api.github.com/repos/willhite/noms/issues/events{/number}","events_url":"https://api.github.com/repos/willhite/noms/events","assignees_url":"https://api.github.com/repos/willhite/noms/assignees{/user}","branches_url":"https://api.github.com/repos/willhite/noms/branches{/branch}","tags_url":"https://api.github.com/repos/willhite/noms/tags","blobs_url":"https://api.github.com/repos/willhite/noms/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/willhite/noms/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/willhite/noms/git/refs{/sha}","trees_url":"https://api.github.com/repos/willhite/noms/git/trees{/sha}","statuses_url":"https://api.github.com/repos/willhite/noms/statuses/{sha}","languages_url":"https://api.github.com/repos/willhite/noms/languages","stargazers_url":"https://api.github.com/repos/willhite/noms/stargazers","contributors_url":"https://api.github.com/repos/willhite/noms/contributors","subscribers_url":"https://api.github.com/repos/willhite/noms/subscribers","subscription_url":"https://api.github.com/repos/willhite/noms/subscription","commits_url":"https://api.github.com/repos/willhite/noms/commits{/sha}","git_commits_url":"https://api.github.com/repos/willhite/noms/git/commits{/sha}","comments_url":"https://api.github.com/repos/willhite/noms/comments{/number}","issue_comment_url":"https://api.github.com/repos/willhite/noms/issues/comments{/number}","contents_url":"https://api.github.com/repos/willhite/noms/contents/{+path}","compare_url":"https://api.github.com/repos/willhite/noms/compare/{base}...{head}","merges_url":"https://api.github.com/repos/willhite/noms/merges","archive_url":"https://api.github.com/repos/willhite/noms/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/willhite/noms/downloads","issues_url":"https://api.github.com/repos/willhite/noms/issues{/number}","pulls_url":"https://api.github.com/repos/willhite/noms/pulls{/number}","milestones_url":"https://api.github.com/repos/willhite/noms/milestones{/number}","notifications_url":"https://api.github.com/repos/willhite/noms/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/willhite/noms/labels{/name}","releases_url":"https://api.github.com/repos/willhite/noms/releases{/id}","deployments_url":"https://api.github.com/repos/willhite/noms/deployments","created_at":"2016-08-02T17:25:18Z","updated_at":"2017-02-23T19:44:45Z","pushed_at":"2017-09-06T00:41:55Z","git_url":"git://github.com/willhite/noms.git","ssh_url":"git@github.com:willhite/noms.git","clone_url":"https://github.com/willhite/noms.git","svn_url":"https://github.com/willhite/noms","homepage":"","size":60879,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"attic-labs:master","ref":"master","sha":"8c4fa02c6e92b9d1c103f465db0ef376e6a2471c","user":{"login":"attic-labs","id":12534679,"avatar_url":"https://avatars2.githubusercontent.com/u/12534679?v=4","gravatar_id":"","url":"https://api.github.com/users/attic-labs","html_url":"https://github.com/attic-labs","followers_url":"https://api.github.com/users/attic-labs/followers","following_url":"https://api.github.com/users/attic-labs/following{/other_user}","gists_url":"https://api.github.com/users/attic-labs/gists{/gist_id}","starred_url":"https://api.github.com/users/attic-labs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/attic-labs/subscriptions","organizations_url":"https://api.github.com/users/attic-labs/orgs","repos_url":"https://api.github.com/users/attic-labs/repos","events_url":"https://api.github.com/users/attic-labs/events{/privacy}","received_events_url":"https://api.github.com/users/attic-labs/received_events","type":"Organization","site_admin":false},"repo":{"id":36778364,"name":"noms","full_name":"attic-labs/noms","owner":{"login":"attic-labs","id":12534679,"avatar_url":"https://avatars2.githubusercontent.com/u/12534679?v=4","gravatar_id":"","url":"https://api.github.com/users/attic-labs","html_url":"https://github.com/attic-labs","followers_url":"https://api.github.com/users/attic-labs/followers","following_url":"https://api.github.com/users/attic-labs/following{/other_user}","gists_url":"https://api.github.com/users/attic-labs/gists{/gist_id}","starred_url":"https://api.github.com/users/attic-labs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/attic-labs/subscriptions","organizations_url":"https://api.github.com/users/attic-labs/orgs","repos_url":"https://api.github.com/users/attic-labs/repos","events_url":"https://api.github.com/users/attic-labs/events{/privacy}","received_events_url":"https://api.github.com/users/attic-labs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/attic-labs/noms","description":"The versioned, forkable, syncable database","fork":false,"url":"https://api.github.com/repos/attic-labs/noms","forks_url":"https://api.github.com/repos/attic-labs/noms/forks","keys_url":"https://api.github.com/repos/attic-labs/noms/keys{/key_id}","collaborators_url":"https://api.github.com/repos/attic-labs/noms/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/attic-labs/noms/teams","hooks_url":"https://api.github.com/repos/attic-labs/noms/hooks","issue_events_url":"https://api.github.com/repos/attic-labs/noms/issues/events{/number}","events_url":"https://api.github.com/repos/attic-labs/noms/events","assignees_url":"https://api.github.com/repos/attic-labs/noms/assignees{/user}","branches_url":"https://api.github.com/repos/attic-labs/noms/branches{/branch}","tags_url":"https://api.github.com/repos/attic-labs/noms/tags","blobs_url":"https://api.github.com/repos/attic-labs/noms/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/attic-labs/noms/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/attic-labs/noms/git/refs{/sha}","trees_url":"https://api.github.com/repos/attic-labs/noms/git/trees{/sha}","statuses_url":"https://api.github.com/repos/attic-labs/noms/statuses/{sha}","languages_url":"https://api.github.com/repos/attic-labs/noms/languages","stargazers_url":"https://api.github.com/repos/attic-labs/noms/stargazers","contributors_url":"https://api.github.com/repos/attic-labs/noms/contributors","subscribers_url":"https://api.github.com/repos/attic-labs/noms/subscribers","subscription_url":"https://api.github.com/repos/attic-labs/noms/subscription","commits_url":"https://api.github.com/repos/attic-labs/noms/commits{/sha}","git_commits_url":"https://api.github.com/repos/attic-labs/noms/git/commits{/sha}","comments_url":"https://api.github.com/repos/attic-labs/noms/comments{/number}","issue_comment_url":"https://api.github.com/repos/attic-labs/noms/issues/comments{/number}","contents_url":"https://api.github.com/repos/attic-labs/noms/contents/{+path}","compare_url":"https://api.github.com/repos/attic-labs/noms/compare/{base}...{head}","merges_url":"https://api.github.com/repos/attic-labs/noms/merges","archive_url":"https://api.github.com/repos/attic-labs/noms/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/attic-labs/noms/downloads","issues_url":"https://api.github.com/repos/attic-labs/noms/issues{/number}","pulls_url":"https://api.github.com/repos/attic-labs/noms/pulls{/number}","milestones_url":"https://api.github.com/repos/attic-labs/noms/milestones{/number}","notifications_url":"https://api.github.com/repos/attic-labs/noms/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/attic-labs/noms/labels{/name}","releases_url":"https://api.github.com/repos/attic-labs/noms/releases{/id}","deployments_url":"https://api.github.com/repos/attic-labs/noms/deployments","created_at":"2015-06-03T03:44:49Z","updated_at":"2017-09-05T08:56:12Z","pushed_at":"2017-09-06T00:41:56Z","git_url":"git://github.com/attic-labs/noms.git","ssh_url":"git@github.com:attic-labs/noms.git","clone_url":"https://github.com/attic-labs/noms.git","svn_url":"https://github.com/attic-labs/noms","homepage":"","size":60959,"stargazers_count":6191,"watchers_count":6191,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":187,"mirror_url":null,"open_issues_count":257,"forks":187,"open_issues":257,"watchers":6191,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/attic-labs/noms/pulls/3666"},"html":{"href":"https://github.com/attic-labs/noms/pull/3666"},"issue":{"href":"https://api.github.com/repos/attic-labs/noms/issues/3666"},"comments":{"href":"https://api.github.com/repos/attic-labs/noms/issues/3666/comments"},"review_comments":{"href":"https://api.github.com/repos/attic-labs/noms/pulls/3666/comments"},"review_comment":{"href":"https://api.github.com/repos/attic-labs/noms/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/attic-labs/noms/pulls/3666/commits"},"statuses":{"href":"https://api.github.com/repos/attic-labs/noms/statuses/45c9c23b840e9b24f2436a445d5c5701a2316165"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"willhite"}}