{"action":"created","comment":{"url":"https://api.github.com/repos/m3db/m3collector/pulls/comments/109494831","pull_request_review_id":30555279,"id":109494831,"diff_hunk":"@@ -0,0 +1,177 @@\n+// Copyright (c) 2017 Uber Technologies, Inc.\n+//\n+// Permission is hereby granted, free of charge, to any person obtaining a copy\n+// of this software and associated documentation files (the \"Software\"), to deal\n+// in the Software without restriction, including without limitation the rights\n+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n+// copies of the Software, and to permit persons to whom the Software is\n+// furnished to do so, subject to the following conditions:\n+//\n+// The above copyright notice and this permission notice shall be included in\n+// all copies or substantial portions of the Software.\n+//\n+// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n+// THE SOFTWARE.\n+\n+package rules\n+\n+import (\n+\t\"errors\"\n+\t\"sync\"\n+\t\"time\"\n+\n+\t\"github.com/m3db/m3cluster/kv\"\n+\t\"github.com/m3db/m3collector/runtime\"\n+\t\"github.com/m3db/m3metrics/generated/proto/schema\"\n+\t\"github.com/m3db/m3metrics/rules\"\n+\t\"github.com/m3db/m3x/errors\"\n+\t\"github.com/m3db/m3x/log\"\n+)\n+\n+var (\n+\temptyNamespaces rules.Namespaces\n+\terrNilValue     = errors.New(\"nil value received\")\n+)\n+\n+// namespaces contains the list of namespace users have defined\n+// rules for. The namespace list is updated as needed.\n+type namespaces struct {\n+\tsync.RWMutex\n+\truntime.Value\n+\n+\tkey          string\n+\tstore        kv.Store\n+\tcache        Cache\n+\topts         Options\n+\tlog          xlog.Logger\n+\truleSetKeyFn RuleSetKeyFn\n+\n+\tproto *schema.Namespaces\n+\trules map[string]*ruleSets\n+}\n+\n+func newNamespaces(key string, cache Cache, opts Options) *namespaces {\n+\tn := &namespaces{\n+\t\tkey:          key,\n+\t\tstore:        opts.KVStore(),\n+\t\tcache:        cache,\n+\t\topts:         opts,\n+\t\tlog:          opts.InstrumentOptions().Logger(),\n+\t\truleSetKeyFn: opts.RuleSetKeyFn(),\n+\t\tproto:        &schema.Namespaces{},\n+\t\trules:        make(map[string]*ruleSets),\n+\t}\n+\tvalueOpts := runtime.NewOptions().\n+\t\tSetClockOptions(opts.ClockOptions()).\n+\t\tSetInstrumentOptions(opts.InstrumentOptions()).\n+\t\tSetInitWatchTimeout(opts.InitWatchTimeout()).\n+\t\tSetKVStore(n.store).\n+\t\tSetUnmarshalFn(n.toNamespaces).\n+\t\tSetVersionFn(namespacesVersion).\n+\t\tSetCutoverFn(namespacesCutover).\n+\t\tSetProcessFn(n.process)\n+\tn.Value = runtime.NewValue(key, valueOpts)\n+\treturn n\n+}\n+\n+func (n *namespaces) Close() error {\n+\tmultiErr := xerrors.NewMultiError()\n+\tif err := n.Value.Close(); err != nil {\n+\t\tmultiErr = multiErr.Add(err)\n+\t}\n+\n+\tn.RLock()\n+\tdefer n.RUnlock()\n+\tfor _, rs := range n.rules {\n+\t\tif err := rs.Close(); err != nil {\n+\t\t\tmultiErr = multiErr.Add(err)\n+\t\t}\n+\t}\n+\treturn multiErr.FinalError()\n+}\n+\n+func (n *namespaces) toNamespaces(value kv.Value) (interface{}, error) {\n+\tif value == nil {\n+\t\treturn emptyNamespaces, errNilValue\n+\t}\n+\tn.proto.Reset()\n+\tif err := value.Unmarshal(n.proto); err != nil {\n+\t\treturn emptyNamespaces, err\n+\t}\n+\treturn rules.NewNamespaces(n.proto), nil\n+}\n+\n+func (n *namespaces) process(value interface{}) {\n+\tvar (\n+\t\tnss        = value.(rules.Namespaces)\n+\t\tnamespaces = nss.Namespaces()\n+\t\tincoming   = make(map[string]struct{}, len(namespaces))\n+\t\tadded      = make(map[string]*ruleSets)\n+\t)\n+\tfor _, ns := range namespaces {\n+\t\tincoming[ns] = struct{}{}\n+\t}\n+\n+\tn.RLock()\n+\t// NB(xichen): if a namespace is deleted, we don't remove it from\n+\t// the map right away because there may be rules in it that don't\n+\t// become invalid until certain time in the future. Therefore we\n+\t// simply mark the rulesets for that namespace deleted and only\n+\t// remove it from the rules map when the rules have expired.\n+\tfor ns, existing := range n.rules {\n+\t\tif _, ok := incoming[ns]; !ok {\n+\t\t\tif err := existing.MarkTombstoned(); err != nil {\n+\t\t\t\tn.log.WithFields(\n+\t\t\t\t\txlog.NewLogField(\"ruleSetKey\", existing.Key()),\n+\t\t\t\t\txlog.NewLogErrField(err),\n+\t\t\t\t).Error(\"failed to mark ruleset as tombstoned\")\n+\t\t\t}","path":"rules/namespaces.go","position":133,"original_position":133,"commit_id":"e44d93ae31fdfd61c3caadf9054153c88605b618","original_commit_id":"e44d93ae31fdfd61c3caadf9054153c88605b618","user":{"login":"xichen2020","id":12239421,"avatar_url":"https://avatars2.githubusercontent.com/u/12239421?v=3","gravatar_id":"","url":"https://api.github.com/users/xichen2020","html_url":"https://github.com/xichen2020","followers_url":"https://api.github.com/users/xichen2020/followers","following_url":"https://api.github.com/users/xichen2020/following{/other_user}","gists_url":"https://api.github.com/users/xichen2020/gists{/gist_id}","starred_url":"https://api.github.com/users/xichen2020/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xichen2020/subscriptions","organizations_url":"https://api.github.com/users/xichen2020/orgs","repos_url":"https://api.github.com/users/xichen2020/repos","events_url":"https://api.github.com/users/xichen2020/events{/privacy}","received_events_url":"https://api.github.com/users/xichen2020/received_events","type":"User","site_admin":false},"body":"Sure","created_at":"2017-04-03T18:45:35Z","updated_at":"2017-04-03T18:58:50Z","html_url":"https://github.com/m3db/m3collector/pull/2#discussion_r109494831","pull_request_url":"https://api.github.com/repos/m3db/m3collector/pulls/2","_links":{"self":{"href":"https://api.github.com/repos/m3db/m3collector/pulls/comments/109494831"},"html":{"href":"https://github.com/m3db/m3collector/pull/2#discussion_r109494831"},"pull_request":{"href":"https://api.github.com/repos/m3db/m3collector/pulls/2"}}},"pull_request":{"url":"https://api.github.com/repos/m3db/m3collector/pulls/2","id":110220554,"html_url":"https://github.com/m3db/m3collector/pull/2","diff_url":"https://github.com/m3db/m3collector/pull/2.diff","patch_url":"https://github.com/m3db/m3collector/pull/2.patch","issue_url":"https://api.github.com/repos/m3db/m3collector/issues/2","number":2,"state":"open","locked":false,"title":"Ruleset propagation and matching logic","user":{"login":"xichen2020","id":12239421,"avatar_url":"https://avatars2.githubusercontent.com/u/12239421?v=3","gravatar_id":"","url":"https://api.github.com/users/xichen2020","html_url":"https://github.com/xichen2020","followers_url":"https://api.github.com/users/xichen2020/followers","following_url":"https://api.github.com/users/xichen2020/following{/other_user}","gists_url":"https://api.github.com/users/xichen2020/gists{/gist_id}","starred_url":"https://api.github.com/users/xichen2020/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xichen2020/subscriptions","organizations_url":"https://api.github.com/users/xichen2020/orgs","repos_url":"https://api.github.com/users/xichen2020/repos","events_url":"https://api.github.com/users/xichen2020/events{/privacy}","received_events_url":"https://api.github.com/users/xichen2020/received_events","type":"User","site_admin":false},"body":"cc @robskillington @cw9 @prateek @jeromefroe \r\n\r\nThis PR adds logic to handle ruleset propagation, including watching and processing ruleset updates, as well as rule key changes. It then uses the rules to matching against metric IDs to determine the applicable policies.\r\n\r\nThere are a few interfaces (namely `BackendServer`, `Reporter`, and `Matcher`) that don't have concrete implementations yet. Just want to get them out there to get some early feedback. \r\n\r\nI'm going to work on adding concrete implementations for the matcher interface (e.g., caching), and hooking up the matcher to the reporter, which talks to the backend server that forwards metrics to the aggregation tier.","created_at":"2017-03-10T22:19:17Z","updated_at":"2017-04-03T18:58:50Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a8f5120897e803cbcecf50dde06a131bf4f238fc","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/m3db/m3collector/pulls/2/commits","review_comments_url":"https://api.github.com/repos/m3db/m3collector/pulls/2/comments","review_comment_url":"https://api.github.com/repos/m3db/m3collector/pulls/comments{/number}","comments_url":"https://api.github.com/repos/m3db/m3collector/issues/2/comments","statuses_url":"https://api.github.com/repos/m3db/m3collector/statuses/e44d93ae31fdfd61c3caadf9054153c88605b618","head":{"label":"m3db:xichen-ruleset-propagation","ref":"xichen-ruleset-propagation","sha":"e44d93ae31fdfd61c3caadf9054153c88605b618","user":{"login":"m3db","id":19937171,"avatar_url":"https://avatars3.githubusercontent.com/u/19937171?v=3","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"repo":{"id":79852610,"name":"m3collector","full_name":"m3db/m3collector","owner":{"login":"m3db","id":19937171,"avatar_url":"https://avatars3.githubusercontent.com/u/19937171?v=3","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/m3db/m3collector","description":"Collects local metrics and sends them to downstream services","fork":false,"url":"https://api.github.com/repos/m3db/m3collector","forks_url":"https://api.github.com/repos/m3db/m3collector/forks","keys_url":"https://api.github.com/repos/m3db/m3collector/keys{/key_id}","collaborators_url":"https://api.github.com/repos/m3db/m3collector/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/m3db/m3collector/teams","hooks_url":"https://api.github.com/repos/m3db/m3collector/hooks","issue_events_url":"https://api.github.com/repos/m3db/m3collector/issues/events{/number}","events_url":"https://api.github.com/repos/m3db/m3collector/events","assignees_url":"https://api.github.com/repos/m3db/m3collector/assignees{/user}","branches_url":"https://api.github.com/repos/m3db/m3collector/branches{/branch}","tags_url":"https://api.github.com/repos/m3db/m3collector/tags","blobs_url":"https://api.github.com/repos/m3db/m3collector/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/m3db/m3collector/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/m3db/m3collector/git/refs{/sha}","trees_url":"https://api.github.com/repos/m3db/m3collector/git/trees{/sha}","statuses_url":"https://api.github.com/repos/m3db/m3collector/statuses/{sha}","languages_url":"https://api.github.com/repos/m3db/m3collector/languages","stargazers_url":"https://api.github.com/repos/m3db/m3collector/stargazers","contributors_url":"https://api.github.com/repos/m3db/m3collector/contributors","subscribers_url":"https://api.github.com/repos/m3db/m3collector/subscribers","subscription_url":"https://api.github.com/repos/m3db/m3collector/subscription","commits_url":"https://api.github.com/repos/m3db/m3collector/commits{/sha}","git_commits_url":"https://api.github.com/repos/m3db/m3collector/git/commits{/sha}","comments_url":"https://api.github.com/repos/m3db/m3collector/comments{/number}","issue_comment_url":"https://api.github.com/repos/m3db/m3collector/issues/comments{/number}","contents_url":"https://api.github.com/repos/m3db/m3collector/contents/{+path}","compare_url":"https://api.github.com/repos/m3db/m3collector/compare/{base}...{head}","merges_url":"https://api.github.com/repos/m3db/m3collector/merges","archive_url":"https://api.github.com/repos/m3db/m3collector/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/m3db/m3collector/downloads","issues_url":"https://api.github.com/repos/m3db/m3collector/issues{/number}","pulls_url":"https://api.github.com/repos/m3db/m3collector/pulls{/number}","milestones_url":"https://api.github.com/repos/m3db/m3collector/milestones{/number}","notifications_url":"https://api.github.com/repos/m3db/m3collector/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/m3db/m3collector/labels{/name}","releases_url":"https://api.github.com/repos/m3db/m3collector/releases{/id}","deployments_url":"https://api.github.com/repos/m3db/m3collector/deployments","created_at":"2017-01-23T21:59:30Z","updated_at":"2017-01-25T16:59:23Z","pushed_at":"2017-03-30T18:18:48Z","git_url":"git://github.com/m3db/m3collector.git","ssh_url":"git@github.com:m3db/m3collector.git","clone_url":"https://github.com/m3db/m3collector.git","svn_url":"https://github.com/m3db/m3collector","homepage":null,"size":150,"stargazers_count":0,"watchers_count":0,"language":"Makefile","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"m3db:master","ref":"master","sha":"3f4ba689e07a65fdf745e720968b4e8b7171e1f1","user":{"login":"m3db","id":19937171,"avatar_url":"https://avatars3.githubusercontent.com/u/19937171?v=3","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"repo":{"id":79852610,"name":"m3collector","full_name":"m3db/m3collector","owner":{"login":"m3db","id":19937171,"avatar_url":"https://avatars3.githubusercontent.com/u/19937171?v=3","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/m3db/m3collector","description":"Collects local metrics and sends them to downstream services","fork":false,"url":"https://api.github.com/repos/m3db/m3collector","forks_url":"https://api.github.com/repos/m3db/m3collector/forks","keys_url":"https://api.github.com/repos/m3db/m3collector/keys{/key_id}","collaborators_url":"https://api.github.com/repos/m3db/m3collector/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/m3db/m3collector/teams","hooks_url":"https://api.github.com/repos/m3db/m3collector/hooks","issue_events_url":"https://api.github.com/repos/m3db/m3collector/issues/events{/number}","events_url":"https://api.github.com/repos/m3db/m3collector/events","assignees_url":"https://api.github.com/repos/m3db/m3collector/assignees{/user}","branches_url":"https://api.github.com/repos/m3db/m3collector/branches{/branch}","tags_url":"https://api.github.com/repos/m3db/m3collector/tags","blobs_url":"https://api.github.com/repos/m3db/m3collector/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/m3db/m3collector/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/m3db/m3collector/git/refs{/sha}","trees_url":"https://api.github.com/repos/m3db/m3collector/git/trees{/sha}","statuses_url":"https://api.github.com/repos/m3db/m3collector/statuses/{sha}","languages_url":"https://api.github.com/repos/m3db/m3collector/languages","stargazers_url":"https://api.github.com/repos/m3db/m3collector/stargazers","contributors_url":"https://api.github.com/repos/m3db/m3collector/contributors","subscribers_url":"https://api.github.com/repos/m3db/m3collector/subscribers","subscription_url":"https://api.github.com/repos/m3db/m3collector/subscription","commits_url":"https://api.github.com/repos/m3db/m3collector/commits{/sha}","git_commits_url":"https://api.github.com/repos/m3db/m3collector/git/commits{/sha}","comments_url":"https://api.github.com/repos/m3db/m3collector/comments{/number}","issue_comment_url":"https://api.github.com/repos/m3db/m3collector/issues/comments{/number}","contents_url":"https://api.github.com/repos/m3db/m3collector/contents/{+path}","compare_url":"https://api.github.com/repos/m3db/m3collector/compare/{base}...{head}","merges_url":"https://api.github.com/repos/m3db/m3collector/merges","archive_url":"https://api.github.com/repos/m3db/m3collector/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/m3db/m3collector/downloads","issues_url":"https://api.github.com/repos/m3db/m3collector/issues{/number}","pulls_url":"https://api.github.com/repos/m3db/m3collector/pulls{/number}","milestones_url":"https://api.github.com/repos/m3db/m3collector/milestones{/number}","notifications_url":"https://api.github.com/repos/m3db/m3collector/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/m3db/m3collector/labels{/name}","releases_url":"https://api.github.com/repos/m3db/m3collector/releases{/id}","deployments_url":"https://api.github.com/repos/m3db/m3collector/deployments","created_at":"2017-01-23T21:59:30Z","updated_at":"2017-01-25T16:59:23Z","pushed_at":"2017-03-30T18:18:48Z","git_url":"git://github.com/m3db/m3collector.git","ssh_url":"git@github.com:m3db/m3collector.git","clone_url":"https://github.com/m3db/m3collector.git","svn_url":"https://github.com/m3db/m3collector","homepage":null,"size":150,"stargazers_count":0,"watchers_count":0,"language":"Makefile","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/m3db/m3collector/pulls/2"},"html":{"href":"https://github.com/m3db/m3collector/pull/2"},"issue":{"href":"https://api.github.com/repos/m3db/m3collector/issues/2"},"comments":{"href":"https://api.github.com/repos/m3db/m3collector/issues/2/comments"},"review_comments":{"href":"https://api.github.com/repos/m3db/m3collector/pulls/2/comments"},"review_comment":{"href":"https://api.github.com/repos/m3db/m3collector/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/m3db/m3collector/pulls/2/commits"},"statuses":{"href":"https://api.github.com/repos/m3db/m3collector/statuses/e44d93ae31fdfd61c3caadf9054153c88605b618"}}}}
{"actor":{"display_login":"xichen2020"}}