{"action":"created","comment":{"url":"https://api.github.com/repos/juju/juju/pulls/comments/155622914","pull_request_review_id":81973286,"id":155622914,"diff_hunk":"@@ -0,0 +1,317 @@\n+// Copyright 2012, 2013 Canonical Ltd.\n+// Licensed under the AGPLv3, see LICENCE file for details.\n+\n+package watcher\n+\n+import (\n+\t\"strings\"\n+\t\"time\"\n+\n+\t\"github.com/juju/errors\"\n+\t\"gopkg.in/juju/worker.v1\"\n+\t\"gopkg.in/mgo.v2\"\n+\t\"gopkg.in/mgo.v2/bson\"\n+\t\"gopkg.in/retry.v1\"\n+\t\"gopkg.in/tomb.v1\"\n+\n+\t\"github.com/juju/juju/mongo\"\n+)\n+\n+// Hub represents a pubsub hub. The TxnWatcher only ever publishes\n+// events to the hub.\n+type Hub interface {\n+\tPublish(topic string, data interface{}) <-chan struct{}\n+}\n+\n+// Clock represents the time methods used.\n+type Clock interface {\n+\tNow() time.Time\n+\tAfter(time.Duration) <-chan time.Time\n+}\n+\n+const (\n+\ttxnWatcherStarting   = \"starting\"\n+\ttxnWatcherCollection = \"collection\"\n+\n+\ttxnWatcherShortWait = 10 * time.Millisecond\n+)\n+\n+// PollStrategy is used to determine how long\n+// to delay between poll intervals. A new timer\n+// is created each time some watcher event is\n+// fired or if the old timer completes.\n+//\n+// It must not be changed when any watchers are active.\n+var PollStrategy retry.Strategy = retry.Exponential{\n+\tInitial:  10 * time.Millisecond,\n+\tFactor:   1.5,\n+\tMaxDelay: 5 * time.Second,\n+}\n+\n+type txnChange struct {\n+\tcollection string\n+\tdocID      interface{}\n+\trevID      int64\n+}\n+\n+// A TxnWatcher watches the txns.log collection and publishes all change events\n+// to the hub.\n+type TxnWatcher struct {\n+\thub    Hub\n+\tclock  Clock\n+\tlogger Logger\n+\n+\ttomb         tomb.Tomb\n+\titeratorFunc func() mongo.Iterator\n+\tlog          *mgo.Collection\n+\n+\t// syncEvents contain the events to be\n+\t// dispatched to the watcher channels. They're queued during\n+\t// processing and flushed at the end to simplify the algorithm.\n+\t// The two queues are separated because events from sync are\n+\t// handled in reverse order due to the way the algorithm works.\n+\tsyncEvents []Change\n+\n+\t// lastId is the most recent transaction id observed by a sync.\n+\tlastId interface{}\n+}\n+\n+// TxnWatcherConfig container the configuration parameters required\n+// for a NewTxnWatcher.\n+type TxnWatcherConfig struct {\n+\t// ChangeLog is usually the tnxs.log collection.\n+\tChangeLog *mgo.Collection\n+\t// Hub is where the changes are published to.\n+\tHub Hub\n+\t// Clock allows tests to control the advancing of time.\n+\tClock Clock\n+\t// Logger is used to control where the log messages for this watcher go.\n+\tLogger Logger\n+\t// IteratorFunc can be overridden in tests to control what values the\n+\t// watcher sees.\n+\tIteratorFunc func() mongo.Iterator\n+}\n+\n+// Validate ensures that all the values that have to be set are set.\n+func (config TxnWatcherConfig) Validate() error {\n+\tif config.ChangeLog == nil {\n+\t\treturn errors.NotValidf(\"missing ChangeLog\")\n+\t}\n+\tif config.Hub == nil {\n+\t\treturn errors.NotValidf(\"missing Hub\")\n+\t}\n+\tif config.Clock == nil {\n+\t\treturn errors.NotValidf(\"missing Clock\")\n+\t}\n+\treturn nil\n+}\n+\n+// New returns a new Watcher observing the changelog collection,\n+// which must be a capped collection maintained by mgo/txn.\n+func NewTxnWatcher(config TxnWatcherConfig) (*TxnWatcher, error) {\n+\tif err := config.Validate(); err != nil {\n+\t\treturn nil, errors.Annotate(err, \"new TxnWatcher invalid config\")\n+\t}\n+\n+\tw := &TxnWatcher{\n+\t\thub:          config.Hub,\n+\t\tclock:        config.Clock,\n+\t\tlogger:       config.Logger,\n+\t\tlog:          config.ChangeLog,\n+\t\titeratorFunc: config.IteratorFunc,\n+\t}\n+\tif w.iteratorFunc == nil {\n+\t\tw.iteratorFunc = w.iter\n+\t}\n+\tif w.logger == nil {\n+\t\tw.logger = noOpLogger{}\n+\t}\n+\tgo func() {\n+\t\terr := w.loop()\n+\t\tcause := errors.Cause(err)\n+\t\t// tomb expects ErrDying or ErrStillAlive as\n+\t\t// exact values, so we need to log and unwrap\n+\t\t// the error first.\n+\t\tif err != nil && cause != tomb.ErrDying {\n+\t\t\tw.logger.Infof(\"watcher loop failed: %v\", err)\n+\t\t}\n+\t\tw.tomb.Kill(cause)\n+\t\tw.tomb.Done()\n+\t}()\n+\treturn w, nil\n+}\n+\n+// Kill is part of the worker.Worker interface.\n+func (w *TxnWatcher) Kill() {\n+\tw.tomb.Kill(nil)\n+}\n+\n+// Wait is part of the worker.Worker interface.\n+func (w *TxnWatcher) Wait() error {\n+\treturn w.tomb.Wait()\n+}\n+\n+// Stop stops all the watcher activities.\n+func (w *TxnWatcher) Stop() error {\n+\treturn worker.Stop(w)\n+}\n+\n+// Dead returns a channel that is closed when the watcher has stopped.\n+func (w *TxnWatcher) Dead() <-chan struct{} {\n+\treturn w.tomb.Dead()\n+}\n+\n+// Err returns the error with which the watcher stopped.\n+// It returns nil if the watcher stopped cleanly, tomb.ErrStillAlive\n+// if the watcher is still running properly, or the respective error\n+// if the watcher is terminating or has terminated with an error.\n+func (w *TxnWatcher) Err() error {\n+\treturn w.tomb.Err()\n+}\n+\n+// loop implements the main watcher loop.\n+// period is the delay between each sync.\n+func (w *TxnWatcher) loop() error {\n+\tif err := w.initLastId(); err != nil {\n+\t\treturn errors.Trace(err)\n+\t}\n+\t// Make sure we have read the last ID before telling people\n+\t// we have started.\n+\tw.hub.Publish(txnWatcherStarting, nil)\n+\tbackoff := PollStrategy.NewTimer(w.clock.Now())\n+\tnext := w.clock.After(txnWatcherShortWait)\n+\tfor {\n+\t\tselect {\n+\t\tcase <-w.tomb.Dying():\n+\t\t\treturn errors.Trace(tomb.ErrDying)\n+\t\tcase <-next:\n+\t\t\td, ok := backoff.NextSleep(w.clock.Now())\n+\t\t\tif !ok {\n+\t\t\t\t// This shouldn't happen, but be defensive.\n+\t\t\t\tbackoff = PollStrategy.NewTimer(w.clock.Now())\n+\t\t\t}\n+\t\t\tnext = w.clock.After(d)\n+\t\t}\n+\n+\t\tadded, err := w.sync()\n+\t\tif err != nil {\n+\t\t\treturn errors.Trace(err)\n+\t\t}\n+\t\tw.flush()\n+\t\tif added {\n+\t\t\t// Something's happened, so reset the exponential backoff\n+\t\t\t// so we'll retry again quickly.\n+\t\t\tbackoff = PollStrategy.NewTimer(w.clock.Now())\n+\t\t\tnext = w.clock.After(txnWatcherShortWait)\n+\t\t}\n+\t}\n+}\n+\n+// flush sends all pending events to their respective channels.\n+func (w *TxnWatcher) flush() {\n+\t// refreshEvents are stored newest first.\n+\tfor i := len(w.syncEvents) - 1; i >= 0; i-- {\n+\t\te := w.syncEvents[i]\n+\t\tw.hub.Publish(txnWatcherCollection, e)\n+\t}\n+\tw.syncEvents = w.syncEvents[:0]\n+}\n+\n+// initLastId reads the most recent changelog document and initializes\n+// lastId with it. This causes all history that precedes the creation\n+// of the watcher to be ignored.\n+func (w *TxnWatcher) initLastId() error {\n+\tvar entry struct {\n+\t\tId interface{} `bson:\"_id\"`\n+\t}\n+\terr := w.log.Find(nil).Sort(\"-$natural\").One(&entry)\n+\tif err != nil && err != mgo.ErrNotFound {\n+\t\treturn errors.Trace(err)\n+\t}\n+\tw.lastId = entry.Id\n+\treturn nil\n+}\n+\n+func (w *TxnWatcher) iter() mongo.Iterator {\n+\treturn w.log.Find(nil).Batch(10).Sort(\"-$natural\").Iter()\n+}\n+\n+// sync updates the watcher knowledge from the database, and\n+// queues events to observing channels.\n+func (w *TxnWatcher) sync() (bool, error) {\n+\tadded := false\n+\t// Iterate through log events in reverse insertion order (newest first).\n+\titer := w.iteratorFunc()\n+\tseen := make(map[watchKey]bool)\n+\tfirst := true\n+\tlastId := w.lastId\n+\tvar entry bson.D\n+\tfor iter.Next(&entry) {\n+\t\tif len(entry) == 0 {\n+\t\t\tw.logger.Tracef(\"got empty changelog document\")\n+\t\t}\n+\t\tid := entry[0]\n+\t\tif id.Name != \"_id\" {\n+\t\t\tw.logger.Warningf(\"watcher: _id field isn't first entry\")\n+\t\t\tcontinue\n+\t\t}\n+\t\tif first {\n+\t\t\tw.lastId = id.Value\n+\t\t\tfirst = false\n+\t\t}\n+\t\tif id.Value == lastId {\n+\t\t\tbreak\n+\t\t}\n+\t\tw.logger.Tracef(\"got changelog document: %#v\", entry)\n+\t\tfor _, c := range entry[1:] {\n+\t\t\t// See txn's Runner.ChangeLog for the structure of log entries.\n+\t\t\tvar d, r []interface{}\n+\t\t\tdr, _ := c.Value.(bson.D)\n+\t\t\tfor _, item := range dr {\n+\t\t\t\tswitch item.Name {\n+\t\t\t\tcase \"d\":\n+\t\t\t\t\td, _ = item.Value.([]interface{})\n+\t\t\t\tcase \"r\":\n+\t\t\t\t\tr, _ = item.Value.([]interface{})\n+\t\t\t\t}\n+\t\t\t}\n+\t\t\tif len(d) == 0 || len(d) != len(r) {\n+\t\t\t\tw.logger.Warningf(\"changelog has invalid collection document: %#v\", c)\n+\t\t\t\tcontinue\n+\t\t\t}\n+\t\t\tfor i := len(d) - 1; i >= 0; i-- {\n+\t\t\t\tkey := watchKey{c.Name, d[i]}\n+\t\t\t\tif seen[key] {\n+\t\t\t\t\tcontinue\n+\t\t\t\t}\n+\t\t\t\tseen[key] = true\n+\t\t\t\trevno, ok := r[i].(int64)\n+\t\t\t\tif !ok {\n+\t\t\t\t\tw.logger.Warningf(\"changelog has revno with type %T: %#v\", r[i], r[i])\n+\t\t\t\t\tcontinue\n+\t\t\t\t}\n+\t\t\t\tif revno < 0 {\n+\t\t\t\t\trevno = -1\n+\t\t\t\t}\n+\t\t\t\tw.syncEvents = append(w.syncEvents, Change{\n+\t\t\t\t\tC:     c.Name,\n+\t\t\t\t\tId:    d[i],\n+\t\t\t\t\tRevno: revno,\n+\t\t\t\t})\n+\t\t\t\tadded = true\n+\t\t\t}\n+\t\t}\n+\t}\n+\tif err := iter.Close(); err != nil {\n+\t\tif qerr, ok := err.(*mgo.QueryError); ok {\n+\t\t\t// CappedPositionLost is code 136.\n+\t\t\t// Just in case that changes for some reason, we'll also check the error message.\n+\t\t\tif qerr.Code == 136 || strings.Contains(qerr.Message, \"CappedPositionLost\") {","path":"state/watcher/txnwatcher.go","position":309,"original_position":309,"commit_id":"268ad98f42c9b4d9322fc1c1359148742c3aaf36","original_commit_id":"268ad98f42c9b4d9322fc1c1359148742c3aaf36","user":{"login":"howbazaar","id":1744247,"avatar_url":"https://avatars3.githubusercontent.com/u/1744247?v=4","gravatar_id":"","url":"https://api.github.com/users/howbazaar","html_url":"https://github.com/howbazaar","followers_url":"https://api.github.com/users/howbazaar/followers","following_url":"https://api.github.com/users/howbazaar/following{/other_user}","gists_url":"https://api.github.com/users/howbazaar/gists{/gist_id}","starred_url":"https://api.github.com/users/howbazaar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/howbazaar/subscriptions","organizations_url":"https://api.github.com/users/howbazaar/orgs","repos_url":"https://api.github.com/users/howbazaar/repos","events_url":"https://api.github.com/users/howbazaar/events{/privacy}","received_events_url":"https://api.github.com/users/howbazaar/received_events","type":"User","site_admin":false},"body":"Yeah, it is a bit ick, and to be honest, I don't think we need to care for this now.","created_at":"2017-12-07T19:50:04Z","updated_at":"2017-12-07T19:50:04Z","html_url":"https://github.com/juju/juju/pull/8185#discussion_r155622914","pull_request_url":"https://api.github.com/repos/juju/juju/pulls/8185","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/juju/juju/pulls/comments/155622914"},"html":{"href":"https://github.com/juju/juju/pull/8185#discussion_r155622914"},"pull_request":{"href":"https://api.github.com/repos/juju/juju/pulls/8185"}},"in_reply_to_id":155521979},"pull_request":{"url":"https://api.github.com/repos/juju/juju/pulls/8185","id":156900090,"html_url":"https://github.com/juju/juju/pull/8185","diff_url":"https://github.com/juju/juju/pull/8185.diff","patch_url":"https://github.com/juju/juju/pull/8185.patch","issue_url":"https://api.github.com/repos/juju/juju/issues/8185","number":8185,"state":"open","locked":false,"title":"Make the state pool own the main txn log watcher.","user":{"login":"howbazaar","id":1744247,"avatar_url":"https://avatars3.githubusercontent.com/u/1744247?v=4","gravatar_id":"","url":"https://api.github.com/users/howbazaar","html_url":"https://github.com/howbazaar","followers_url":"https://api.github.com/users/howbazaar/followers","following_url":"https://api.github.com/users/howbazaar/following{/other_user}","gists_url":"https://api.github.com/users/howbazaar/gists{/gist_id}","starred_url":"https://api.github.com/users/howbazaar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/howbazaar/subscriptions","organizations_url":"https://api.github.com/users/howbazaar/orgs","repos_url":"https://api.github.com/users/howbazaar/repos","events_url":"https://api.github.com/users/howbazaar/events{/privacy}","received_events_url":"https://api.github.com/users/howbazaar/received_events","type":"User","site_admin":false},"body":"Back in the dawn of time, there was only a single model, and a single State instance. As the controller concept became a thing, we had additional State instances for each model. A core aspect of how the event model works in Juju is watching of the txns.log collection. As documents are added, updated, and removed by the transaction system, documents are added to the txns.log capped collection. Each document refers to the documents that were touched a single transaction. Each State instance has its own txn watcher. Each one of these tails the txns.log collection to watch for changes. As the number of models grows in a single controller, so does the number of things watching the txns.log collection. If the controller is HA, then there is n x m watchers where n is the number of models and m is the number of API servers. This creates quite a bit of idle load on mongo as the number of models grows above a small number. As numbers approach 200, there is a LOT of load. Especially when actions occur that trigger a bunch of transactions that touch a bunch of documents, such as removing an application. Particularly an application that has a bunch of subordinates.\r\n\r\nSo... this branch changes how the polling of the database is done. The state pool now has a watcher, that publishes all document changes to a hub that is also owned by the state pool. Whenever the state pool creates a new State instance, the base watcher for that State instance listens to the hub for changes rather than polling the database.\r\n\r\nThis drastically reduces the mongo load.\r\n\r\n## QA steps\r\n\r\nI verified this by starting with a 2.2.6 controller, deploying 55 units to each of four models, and adding 700 empty models. This showed significant i/o load on the juju.txns.log collection.\r\n\r\nAfter upgrading the controller, the load dropped significantly. From ~1000ms read / 10s to 15ms read / 10s.\r\n\r\n## Documentation changes\r\n\r\nThis is internal only, no documentation changes are necessary.\r\n\r\n## Bug reference\r\n\r\nhttps://bugs.launchpad.net/juju/+bug/1733708\r\n","created_at":"2017-12-07T04:03:52Z","updated_at":"2017-12-07T19:50:04Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2c3016251836c4cfe69120f32198829dde6d8e70","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/juju/juju/pulls/8185/commits","review_comments_url":"https://api.github.com/repos/juju/juju/pulls/8185/comments","review_comment_url":"https://api.github.com/repos/juju/juju/pulls/comments{/number}","comments_url":"https://api.github.com/repos/juju/juju/issues/8185/comments","statuses_url":"https://api.github.com/repos/juju/juju/statuses/268ad98f42c9b4d9322fc1c1359148742c3aaf36","head":{"label":"howbazaar:2.3-txn-log-watcher-pubsub","ref":"2.3-txn-log-watcher-pubsub","sha":"268ad98f42c9b4d9322fc1c1359148742c3aaf36","user":{"login":"howbazaar","id":1744247,"avatar_url":"https://avatars3.githubusercontent.com/u/1744247?v=4","gravatar_id":"","url":"https://api.github.com/users/howbazaar","html_url":"https://github.com/howbazaar","followers_url":"https://api.github.com/users/howbazaar/followers","following_url":"https://api.github.com/users/howbazaar/following{/other_user}","gists_url":"https://api.github.com/users/howbazaar/gists{/gist_id}","starred_url":"https://api.github.com/users/howbazaar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/howbazaar/subscriptions","organizations_url":"https://api.github.com/users/howbazaar/orgs","repos_url":"https://api.github.com/users/howbazaar/repos","events_url":"https://api.github.com/users/howbazaar/events{/privacy}","received_events_url":"https://api.github.com/users/howbazaar/received_events","type":"User","site_admin":false},"repo":{"id":20467403,"name":"juju","full_name":"howbazaar/juju","owner":{"login":"howbazaar","id":1744247,"avatar_url":"https://avatars3.githubusercontent.com/u/1744247?v=4","gravatar_id":"","url":"https://api.github.com/users/howbazaar","html_url":"https://github.com/howbazaar","followers_url":"https://api.github.com/users/howbazaar/followers","following_url":"https://api.github.com/users/howbazaar/following{/other_user}","gists_url":"https://api.github.com/users/howbazaar/gists{/gist_id}","starred_url":"https://api.github.com/users/howbazaar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/howbazaar/subscriptions","organizations_url":"https://api.github.com/users/howbazaar/orgs","repos_url":"https://api.github.com/users/howbazaar/repos","events_url":"https://api.github.com/users/howbazaar/events{/privacy}","received_events_url":"https://api.github.com/users/howbazaar/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/howbazaar/juju","description":"juju is devops distilled","fork":true,"url":"https://api.github.com/repos/howbazaar/juju","forks_url":"https://api.github.com/repos/howbazaar/juju/forks","keys_url":"https://api.github.com/repos/howbazaar/juju/keys{/key_id}","collaborators_url":"https://api.github.com/repos/howbazaar/juju/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/howbazaar/juju/teams","hooks_url":"https://api.github.com/repos/howbazaar/juju/hooks","issue_events_url":"https://api.github.com/repos/howbazaar/juju/issues/events{/number}","events_url":"https://api.github.com/repos/howbazaar/juju/events","assignees_url":"https://api.github.com/repos/howbazaar/juju/assignees{/user}","branches_url":"https://api.github.com/repos/howbazaar/juju/branches{/branch}","tags_url":"https://api.github.com/repos/howbazaar/juju/tags","blobs_url":"https://api.github.com/repos/howbazaar/juju/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/howbazaar/juju/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/howbazaar/juju/git/refs{/sha}","trees_url":"https://api.github.com/repos/howbazaar/juju/git/trees{/sha}","statuses_url":"https://api.github.com/repos/howbazaar/juju/statuses/{sha}","languages_url":"https://api.github.com/repos/howbazaar/juju/languages","stargazers_url":"https://api.github.com/repos/howbazaar/juju/stargazers","contributors_url":"https://api.github.com/repos/howbazaar/juju/contributors","subscribers_url":"https://api.github.com/repos/howbazaar/juju/subscribers","subscription_url":"https://api.github.com/repos/howbazaar/juju/subscription","commits_url":"https://api.github.com/repos/howbazaar/juju/commits{/sha}","git_commits_url":"https://api.github.com/repos/howbazaar/juju/git/commits{/sha}","comments_url":"https://api.github.com/repos/howbazaar/juju/comments{/number}","issue_comment_url":"https://api.github.com/repos/howbazaar/juju/issues/comments{/number}","contents_url":"https://api.github.com/repos/howbazaar/juju/contents/{+path}","compare_url":"https://api.github.com/repos/howbazaar/juju/compare/{base}...{head}","merges_url":"https://api.github.com/repos/howbazaar/juju/merges","archive_url":"https://api.github.com/repos/howbazaar/juju/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/howbazaar/juju/downloads","issues_url":"https://api.github.com/repos/howbazaar/juju/issues{/number}","pulls_url":"https://api.github.com/repos/howbazaar/juju/pulls{/number}","milestones_url":"https://api.github.com/repos/howbazaar/juju/milestones{/number}","notifications_url":"https://api.github.com/repos/howbazaar/juju/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/howbazaar/juju/labels{/name}","releases_url":"https://api.github.com/repos/howbazaar/juju/releases{/id}","deployments_url":"https://api.github.com/repos/howbazaar/juju/deployments","created_at":"2014-06-04T02:27:18Z","updated_at":"2016-01-15T01:18:35Z","pushed_at":"2017-12-07T04:05:03Z","git_url":"git://github.com/howbazaar/juju.git","ssh_url":"git@github.com:howbazaar/juju.git","clone_url":"https://github.com/howbazaar/juju.git","svn_url":"https://github.com/howbazaar/juju","homepage":null,"size":199863,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"agpl-3.0","name":"GNU Affero General Public License v3.0","spdx_id":"AGPL-3.0","url":"https://api.github.com/licenses/agpl-3.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"juju:2.3","ref":"2.3","sha":"07fa3f05a987674673b0e541cae82171a1c5d16c","user":{"login":"juju","id":4604548,"avatar_url":"https://avatars0.githubusercontent.com/u/4604548?v=4","gravatar_id":"","url":"https://api.github.com/users/juju","html_url":"https://github.com/juju","followers_url":"https://api.github.com/users/juju/followers","following_url":"https://api.github.com/users/juju/following{/other_user}","gists_url":"https://api.github.com/users/juju/gists{/gist_id}","starred_url":"https://api.github.com/users/juju/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/juju/subscriptions","organizations_url":"https://api.github.com/users/juju/orgs","repos_url":"https://api.github.com/users/juju/repos","events_url":"https://api.github.com/users/juju/events{/privacy}","received_events_url":"https://api.github.com/users/juju/received_events","type":"Organization","site_admin":false},"repo":{"id":20445947,"name":"juju","full_name":"juju/juju","owner":{"login":"juju","id":4604548,"avatar_url":"https://avatars0.githubusercontent.com/u/4604548?v=4","gravatar_id":"","url":"https://api.github.com/users/juju","html_url":"https://github.com/juju","followers_url":"https://api.github.com/users/juju/followers","following_url":"https://api.github.com/users/juju/following{/other_user}","gists_url":"https://api.github.com/users/juju/gists{/gist_id}","starred_url":"https://api.github.com/users/juju/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/juju/subscriptions","organizations_url":"https://api.github.com/users/juju/orgs","repos_url":"https://api.github.com/users/juju/repos","events_url":"https://api.github.com/users/juju/events{/privacy}","received_events_url":"https://api.github.com/users/juju/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/juju/juju","description":"juju is devops distilled","fork":false,"url":"https://api.github.com/repos/juju/juju","forks_url":"https://api.github.com/repos/juju/juju/forks","keys_url":"https://api.github.com/repos/juju/juju/keys{/key_id}","collaborators_url":"https://api.github.com/repos/juju/juju/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/juju/juju/teams","hooks_url":"https://api.github.com/repos/juju/juju/hooks","issue_events_url":"https://api.github.com/repos/juju/juju/issues/events{/number}","events_url":"https://api.github.com/repos/juju/juju/events","assignees_url":"https://api.github.com/repos/juju/juju/assignees{/user}","branches_url":"https://api.github.com/repos/juju/juju/branches{/branch}","tags_url":"https://api.github.com/repos/juju/juju/tags","blobs_url":"https://api.github.com/repos/juju/juju/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/juju/juju/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/juju/juju/git/refs{/sha}","trees_url":"https://api.github.com/repos/juju/juju/git/trees{/sha}","statuses_url":"https://api.github.com/repos/juju/juju/statuses/{sha}","languages_url":"https://api.github.com/repos/juju/juju/languages","stargazers_url":"https://api.github.com/repos/juju/juju/stargazers","contributors_url":"https://api.github.com/repos/juju/juju/contributors","subscribers_url":"https://api.github.com/repos/juju/juju/subscribers","subscription_url":"https://api.github.com/repos/juju/juju/subscription","commits_url":"https://api.github.com/repos/juju/juju/commits{/sha}","git_commits_url":"https://api.github.com/repos/juju/juju/git/commits{/sha}","comments_url":"https://api.github.com/repos/juju/juju/comments{/number}","issue_comment_url":"https://api.github.com/repos/juju/juju/issues/comments{/number}","contents_url":"https://api.github.com/repos/juju/juju/contents/{+path}","compare_url":"https://api.github.com/repos/juju/juju/compare/{base}...{head}","merges_url":"https://api.github.com/repos/juju/juju/merges","archive_url":"https://api.github.com/repos/juju/juju/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/juju/juju/downloads","issues_url":"https://api.github.com/repos/juju/juju/issues{/number}","pulls_url":"https://api.github.com/repos/juju/juju/pulls{/number}","milestones_url":"https://api.github.com/repos/juju/juju/milestones{/number}","notifications_url":"https://api.github.com/repos/juju/juju/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/juju/juju/labels{/name}","releases_url":"https://api.github.com/repos/juju/juju/releases{/id}","deployments_url":"https://api.github.com/repos/juju/juju/deployments","created_at":"2014-06-03T14:28:10Z","updated_at":"2017-12-05T23:45:11Z","pushed_at":"2017-12-07T11:21:08Z","git_url":"git://github.com/juju/juju.git","ssh_url":"git@github.com:juju/juju.git","clone_url":"https://github.com/juju/juju.git","svn_url":"https://github.com/juju/juju","homepage":"juju.ubuntu.com","size":204421,"stargazers_count":1062,"watchers_count":1062,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":243,"mirror_url":null,"archived":false,"open_issues_count":14,"license":{"key":"agpl-3.0","name":"GNU Affero General Public License v3.0","spdx_id":"AGPL-3.0","url":"https://api.github.com/licenses/agpl-3.0"},"forks":243,"open_issues":14,"watchers":1062,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/juju/juju/pulls/8185"},"html":{"href":"https://github.com/juju/juju/pull/8185"},"issue":{"href":"https://api.github.com/repos/juju/juju/issues/8185"},"comments":{"href":"https://api.github.com/repos/juju/juju/issues/8185/comments"},"review_comments":{"href":"https://api.github.com/repos/juju/juju/pulls/8185/comments"},"review_comment":{"href":"https://api.github.com/repos/juju/juju/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/juju/juju/pulls/8185/commits"},"statuses":{"href":"https://api.github.com/repos/juju/juju/statuses/268ad98f42c9b4d9322fc1c1359148742c3aaf36"}},"author_association":"OWNER"}}
{"actor":{"display_login":"howbazaar"}}