{"action":"created","comment":{"url":"https://api.github.com/repos/palantir/atlasdb/pulls/comments/129796460","pull_request_review_id":52605325,"id":129796460,"diff_hunk":"@@ -0,0 +1,102 @@\n+/*\n+ * Copyright 2017 Palantir Technologies, Inc. All rights reserved.\n+ *\n+ * Licensed under the BSD-3 License (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ * http://opensource.org/licenses/BSD-3-Clause\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package com.palantir.atlasdb.timelock.clock;\n+\n+import java.time.Duration;\n+import java.time.temporal.ChronoUnit;\n+import java.util.Map;\n+import java.util.Optional;\n+import java.util.Set;\n+import java.util.concurrent.Executors;\n+import java.util.concurrent.ScheduledExecutorService;\n+import java.util.concurrent.TimeUnit;\n+\n+import javax.net.ssl.SSLSocketFactory;\n+\n+import com.google.common.collect.Maps;\n+import com.palantir.atlasdb.http.AtlasDbHttpClients;\n+import com.palantir.atlasdb.util.AtlasDbMetrics;\n+\n+/**\n+ * ClockSkewMonitor keeps track of the system time of the other nodes in the cluster, and compares it to the local\n+ * clock. It's purpose is to monitor if the other nodes' clock progress at the same pace as the local clock.\n+ */\n+public final class ClockSkewMonitor {\n+    private static final Duration PAUSE_BETWEEN_REQUESTS = Duration.of(1, ChronoUnit.SECONDS);\n+\n+    private final ClockSkewEvents events;\n+    private final Map<String, ClockService> monitorByServer;\n+    private final Map<String, RequestTime> previousRequestsByServer;\n+    private final ScheduledExecutorService executorService;\n+    private final ClockService localClockService;\n+\n+    public static ClockSkewMonitor create(Set<String> remoteServers, Optional<SSLSocketFactory> optionalSecurity) {\n+        Map<String, ClockService> monitors = Maps.toMap(remoteServers,\n+                (remoteServer) -> AtlasDbHttpClients.createProxy(optionalSecurity, remoteServer, ClockService.class));\n+\n+        Map<String, RequestTime> previousRequests = Maps.newHashMap();\n+        for (String server : remoteServers) {\n+            previousRequests.put(server, RequestTime.EMPTY);\n+        }\n+\n+        return new ClockSkewMonitor(monitors, previousRequests,\n+                new ClockSkewEvents(AtlasDbMetrics.getMetricRegistry()),\n+                Executors.newSingleThreadScheduledExecutor(),\n+                new ClockServiceImpl());\n+    }\n+\n+    private ClockSkewMonitor(\n+            Map<String, ClockService> monitorByServer,\n+            Map<String, RequestTime> previousRequestsByServer,\n+            ClockSkewEvents events,\n+            ScheduledExecutorService executorService,\n+            ClockService localClockService) {\n+        this.monitorByServer = monitorByServer;\n+        this.previousRequestsByServer = previousRequestsByServer;\n+        this.events = events;\n+        this.executorService = executorService;\n+        this.localClockService = localClockService;\n+    }\n+\n+    public void run() {\n+        executorService.schedule(this::runOnce, PAUSE_BETWEEN_REQUESTS.getSeconds(), TimeUnit.SECONDS);","path":"timelock-server/src/main/java/com/palantir/atlasdb/timelock/clock/ClockSkewMonitor.java","position":76,"original_position":76,"commit_id":"f2557c54b6108b92958a6423a088a837a64dd96a","original_commit_id":"f2557c54b6108b92958a6423a088a837a64dd96a","user":{"login":"nziebart","id":2740512,"avatar_url":"https://avatars1.githubusercontent.com/u/2740512?v=4","gravatar_id":"","url":"https://api.github.com/users/nziebart","html_url":"https://github.com/nziebart","followers_url":"https://api.github.com/users/nziebart/followers","following_url":"https://api.github.com/users/nziebart/following{/other_user}","gists_url":"https://api.github.com/users/nziebart/gists{/gist_id}","starred_url":"https://api.github.com/users/nziebart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nziebart/subscriptions","organizations_url":"https://api.github.com/users/nziebart/orgs","repos_url":"https://api.github.com/users/nziebart/repos","events_url":"https://api.github.com/users/nziebart/events{/privacy}","received_events_url":"https://api.github.com/users/nziebart/received_events","type":"User","site_admin":false},"body":"(`schedule` will just run it once)","created_at":"2017-07-27T09:41:10Z","updated_at":"2017-07-27T09:41:10Z","html_url":"https://github.com/palantir/atlasdb/pull/2176#discussion_r129796460","pull_request_url":"https://api.github.com/repos/palantir/atlasdb/pulls/2176","_links":{"self":{"href":"https://api.github.com/repos/palantir/atlasdb/pulls/comments/129796460"},"html":{"href":"https://github.com/palantir/atlasdb/pull/2176#discussion_r129796460"},"pull_request":{"href":"https://api.github.com/repos/palantir/atlasdb/pulls/2176"}}},"pull_request":{"url":"https://api.github.com/repos/palantir/atlasdb/pulls/2176","id":131141877,"html_url":"https://github.com/palantir/atlasdb/pull/2176","diff_url":"https://github.com/palantir/atlasdb/pull/2176.diff","patch_url":"https://github.com/palantir/atlasdb/pull/2176.patch","issue_url":"https://api.github.com/repos/palantir/atlasdb/issues/2176","number":2176,"state":"open","locked":false,"title":"Clock skew between timelock servers experiment","user":{"login":"fsamuel-bs","id":2558069,"avatar_url":"https://avatars0.githubusercontent.com/u/2558069?v=4","gravatar_id":"","url":"https://api.github.com/users/fsamuel-bs","html_url":"https://github.com/fsamuel-bs","followers_url":"https://api.github.com/users/fsamuel-bs/followers","following_url":"https://api.github.com/users/fsamuel-bs/following{/other_user}","gists_url":"https://api.github.com/users/fsamuel-bs/gists{/gist_id}","starred_url":"https://api.github.com/users/fsamuel-bs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fsamuel-bs/subscriptions","organizations_url":"https://api.github.com/users/fsamuel-bs/orgs","repos_url":"https://api.github.com/users/fsamuel-bs/repos","events_url":"https://api.github.com/users/fsamuel-bs/events{/privacy}","received_events_url":"https://api.github.com/users/fsamuel-bs/received_events","type":"User","site_admin":false},"body":"**Goals (and why)**: We're creating this experiment to investigate the clock skew in production environments. If the clock skew between timelock servers doesn't prove to be too high, we may implement an optimization which involves leasing the leadership for a specified amount of time. Such optimization may avoid the leader verification at each `getFreshTimestamp` call.\r\n\r\n**Implementation Description (bullets)**:\r\n- Add a background thread when the timelock server is created.\r\n- Add an endpoint which exposes the server's system time, in nanos.\r\n- Compare each server skew time.\r\n\r\n**Concerns (what feedback would you like?)**: Is the implementation clean enough? Did I miss any edge case?\r\n\r\n**Where should we start reviewing?**: PaxosTimeLockServer.\r\n\r\n**Priority (whenever / two weeks / yesterday)**: End of week.\r\n\r\n<!---\r\nPlease remember to:\r\n- Add any necessary release notes (including breaking changes)\r\n- Make sure the documentation is up to date for your change\r\n--->\n\n<!-- Reviewable:start -->\n---\nThis change is [<img src=\"https://reviewable.io/review_button.svg\" height=\"34\" align=\"absmiddle\" alt=\"Reviewable\"/>](https://reviewable.io/reviews/palantir/atlasdb/2176)\n<!-- Reviewable:end -->\n","created_at":"2017-07-18T17:17:25Z","updated_at":"2017-07-27T09:41:10Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8a6767b130aa41dd423cf6afc23c09c3a623d1dd","assignee":{"login":"nziebart","id":2740512,"avatar_url":"https://avatars1.githubusercontent.com/u/2740512?v=4","gravatar_id":"","url":"https://api.github.com/users/nziebart","html_url":"https://github.com/nziebart","followers_url":"https://api.github.com/users/nziebart/followers","following_url":"https://api.github.com/users/nziebart/following{/other_user}","gists_url":"https://api.github.com/users/nziebart/gists{/gist_id}","starred_url":"https://api.github.com/users/nziebart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nziebart/subscriptions","organizations_url":"https://api.github.com/users/nziebart/orgs","repos_url":"https://api.github.com/users/nziebart/repos","events_url":"https://api.github.com/users/nziebart/events{/privacy}","received_events_url":"https://api.github.com/users/nziebart/received_events","type":"User","site_admin":false},"assignees":[{"login":"nziebart","id":2740512,"avatar_url":"https://avatars1.githubusercontent.com/u/2740512?v=4","gravatar_id":"","url":"https://api.github.com/users/nziebart","html_url":"https://github.com/nziebart","followers_url":"https://api.github.com/users/nziebart/followers","following_url":"https://api.github.com/users/nziebart/following{/other_user}","gists_url":"https://api.github.com/users/nziebart/gists{/gist_id}","starred_url":"https://api.github.com/users/nziebart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nziebart/subscriptions","organizations_url":"https://api.github.com/users/nziebart/orgs","repos_url":"https://api.github.com/users/nziebart/repos","events_url":"https://api.github.com/users/nziebart/events{/privacy}","received_events_url":"https://api.github.com/users/nziebart/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/palantir/atlasdb/pulls/2176/commits","review_comments_url":"https://api.github.com/repos/palantir/atlasdb/pulls/2176/comments","review_comment_url":"https://api.github.com/repos/palantir/atlasdb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/palantir/atlasdb/issues/2176/comments","statuses_url":"https://api.github.com/repos/palantir/atlasdb/statuses/f2557c54b6108b92958a6423a088a837a64dd96a","head":{"label":"palantir:feature/clock-skew","ref":"feature/clock-skew","sha":"f2557c54b6108b92958a6423a088a837a64dd96a","user":{"login":"palantir","id":303157,"avatar_url":"https://avatars0.githubusercontent.com/u/303157?v=4","gravatar_id":"","url":"https://api.github.com/users/palantir","html_url":"https://github.com/palantir","followers_url":"https://api.github.com/users/palantir/followers","following_url":"https://api.github.com/users/palantir/following{/other_user}","gists_url":"https://api.github.com/users/palantir/gists{/gist_id}","starred_url":"https://api.github.com/users/palantir/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/palantir/subscriptions","organizations_url":"https://api.github.com/users/palantir/orgs","repos_url":"https://api.github.com/users/palantir/repos","events_url":"https://api.github.com/users/palantir/events{/privacy}","received_events_url":"https://api.github.com/users/palantir/received_events","type":"Organization","site_admin":false},"repo":{"id":36960637,"name":"atlasdb","full_name":"palantir/atlasdb","owner":{"login":"palantir","id":303157,"avatar_url":"https://avatars0.githubusercontent.com/u/303157?v=4","gravatar_id":"","url":"https://api.github.com/users/palantir","html_url":"https://github.com/palantir","followers_url":"https://api.github.com/users/palantir/followers","following_url":"https://api.github.com/users/palantir/following{/other_user}","gists_url":"https://api.github.com/users/palantir/gists{/gist_id}","starred_url":"https://api.github.com/users/palantir/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/palantir/subscriptions","organizations_url":"https://api.github.com/users/palantir/orgs","repos_url":"https://api.github.com/users/palantir/repos","events_url":"https://api.github.com/users/palantir/events{/privacy}","received_events_url":"https://api.github.com/users/palantir/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/palantir/atlasdb","description":"Transactional Distributed Database Layer","fork":false,"url":"https://api.github.com/repos/palantir/atlasdb","forks_url":"https://api.github.com/repos/palantir/atlasdb/forks","keys_url":"https://api.github.com/repos/palantir/atlasdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/palantir/atlasdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/palantir/atlasdb/teams","hooks_url":"https://api.github.com/repos/palantir/atlasdb/hooks","issue_events_url":"https://api.github.com/repos/palantir/atlasdb/issues/events{/number}","events_url":"https://api.github.com/repos/palantir/atlasdb/events","assignees_url":"https://api.github.com/repos/palantir/atlasdb/assignees{/user}","branches_url":"https://api.github.com/repos/palantir/atlasdb/branches{/branch}","tags_url":"https://api.github.com/repos/palantir/atlasdb/tags","blobs_url":"https://api.github.com/repos/palantir/atlasdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/palantir/atlasdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/palantir/atlasdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/palantir/atlasdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/palantir/atlasdb/statuses/{sha}","languages_url":"https://api.github.com/repos/palantir/atlasdb/languages","stargazers_url":"https://api.github.com/repos/palantir/atlasdb/stargazers","contributors_url":"https://api.github.com/repos/palantir/atlasdb/contributors","subscribers_url":"https://api.github.com/repos/palantir/atlasdb/subscribers","subscription_url":"https://api.github.com/repos/palantir/atlasdb/subscription","commits_url":"https://api.github.com/repos/palantir/atlasdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/palantir/atlasdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/palantir/atlasdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/palantir/atlasdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/palantir/atlasdb/contents/{+path}","compare_url":"https://api.github.com/repos/palantir/atlasdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/palantir/atlasdb/merges","archive_url":"https://api.github.com/repos/palantir/atlasdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/palantir/atlasdb/downloads","issues_url":"https://api.github.com/repos/palantir/atlasdb/issues{/number}","pulls_url":"https://api.github.com/repos/palantir/atlasdb/pulls{/number}","milestones_url":"https://api.github.com/repos/palantir/atlasdb/milestones{/number}","notifications_url":"https://api.github.com/repos/palantir/atlasdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/palantir/atlasdb/labels{/name}","releases_url":"https://api.github.com/repos/palantir/atlasdb/releases{/id}","deployments_url":"https://api.github.com/repos/palantir/atlasdb/deployments","created_at":"2015-06-05T23:42:44Z","updated_at":"2017-07-24T13:21:07Z","pushed_at":"2017-07-27T09:22:16Z","git_url":"git://github.com/palantir/atlasdb.git","ssh_url":"git@github.com:palantir/atlasdb.git","clone_url":"https://github.com/palantir/atlasdb.git","svn_url":"https://github.com/palantir/atlasdb","homepage":"http://palantir.github.io/atlasdb/","size":42707,"stargazers_count":489,"watchers_count":489,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":84,"mirror_url":null,"open_issues_count":201,"forks":84,"open_issues":201,"watchers":489,"default_branch":"develop"}},"base":{"label":"palantir:develop","ref":"develop","sha":"6607a2a954e696428c5a0b60e6c380b6dfa3817a","user":{"login":"palantir","id":303157,"avatar_url":"https://avatars0.githubusercontent.com/u/303157?v=4","gravatar_id":"","url":"https://api.github.com/users/palantir","html_url":"https://github.com/palantir","followers_url":"https://api.github.com/users/palantir/followers","following_url":"https://api.github.com/users/palantir/following{/other_user}","gists_url":"https://api.github.com/users/palantir/gists{/gist_id}","starred_url":"https://api.github.com/users/palantir/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/palantir/subscriptions","organizations_url":"https://api.github.com/users/palantir/orgs","repos_url":"https://api.github.com/users/palantir/repos","events_url":"https://api.github.com/users/palantir/events{/privacy}","received_events_url":"https://api.github.com/users/palantir/received_events","type":"Organization","site_admin":false},"repo":{"id":36960637,"name":"atlasdb","full_name":"palantir/atlasdb","owner":{"login":"palantir","id":303157,"avatar_url":"https://avatars0.githubusercontent.com/u/303157?v=4","gravatar_id":"","url":"https://api.github.com/users/palantir","html_url":"https://github.com/palantir","followers_url":"https://api.github.com/users/palantir/followers","following_url":"https://api.github.com/users/palantir/following{/other_user}","gists_url":"https://api.github.com/users/palantir/gists{/gist_id}","starred_url":"https://api.github.com/users/palantir/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/palantir/subscriptions","organizations_url":"https://api.github.com/users/palantir/orgs","repos_url":"https://api.github.com/users/palantir/repos","events_url":"https://api.github.com/users/palantir/events{/privacy}","received_events_url":"https://api.github.com/users/palantir/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/palantir/atlasdb","description":"Transactional Distributed Database Layer","fork":false,"url":"https://api.github.com/repos/palantir/atlasdb","forks_url":"https://api.github.com/repos/palantir/atlasdb/forks","keys_url":"https://api.github.com/repos/palantir/atlasdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/palantir/atlasdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/palantir/atlasdb/teams","hooks_url":"https://api.github.com/repos/palantir/atlasdb/hooks","issue_events_url":"https://api.github.com/repos/palantir/atlasdb/issues/events{/number}","events_url":"https://api.github.com/repos/palantir/atlasdb/events","assignees_url":"https://api.github.com/repos/palantir/atlasdb/assignees{/user}","branches_url":"https://api.github.com/repos/palantir/atlasdb/branches{/branch}","tags_url":"https://api.github.com/repos/palantir/atlasdb/tags","blobs_url":"https://api.github.com/repos/palantir/atlasdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/palantir/atlasdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/palantir/atlasdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/palantir/atlasdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/palantir/atlasdb/statuses/{sha}","languages_url":"https://api.github.com/repos/palantir/atlasdb/languages","stargazers_url":"https://api.github.com/repos/palantir/atlasdb/stargazers","contributors_url":"https://api.github.com/repos/palantir/atlasdb/contributors","subscribers_url":"https://api.github.com/repos/palantir/atlasdb/subscribers","subscription_url":"https://api.github.com/repos/palantir/atlasdb/subscription","commits_url":"https://api.github.com/repos/palantir/atlasdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/palantir/atlasdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/palantir/atlasdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/palantir/atlasdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/palantir/atlasdb/contents/{+path}","compare_url":"https://api.github.com/repos/palantir/atlasdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/palantir/atlasdb/merges","archive_url":"https://api.github.com/repos/palantir/atlasdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/palantir/atlasdb/downloads","issues_url":"https://api.github.com/repos/palantir/atlasdb/issues{/number}","pulls_url":"https://api.github.com/repos/palantir/atlasdb/pulls{/number}","milestones_url":"https://api.github.com/repos/palantir/atlasdb/milestones{/number}","notifications_url":"https://api.github.com/repos/palantir/atlasdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/palantir/atlasdb/labels{/name}","releases_url":"https://api.github.com/repos/palantir/atlasdb/releases{/id}","deployments_url":"https://api.github.com/repos/palantir/atlasdb/deployments","created_at":"2015-06-05T23:42:44Z","updated_at":"2017-07-24T13:21:07Z","pushed_at":"2017-07-27T09:22:16Z","git_url":"git://github.com/palantir/atlasdb.git","ssh_url":"git@github.com:palantir/atlasdb.git","clone_url":"https://github.com/palantir/atlasdb.git","svn_url":"https://github.com/palantir/atlasdb","homepage":"http://palantir.github.io/atlasdb/","size":42707,"stargazers_count":489,"watchers_count":489,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":84,"mirror_url":null,"open_issues_count":201,"forks":84,"open_issues":201,"watchers":489,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/palantir/atlasdb/pulls/2176"},"html":{"href":"https://github.com/palantir/atlasdb/pull/2176"},"issue":{"href":"https://api.github.com/repos/palantir/atlasdb/issues/2176"},"comments":{"href":"https://api.github.com/repos/palantir/atlasdb/issues/2176/comments"},"review_comments":{"href":"https://api.github.com/repos/palantir/atlasdb/pulls/2176/comments"},"review_comment":{"href":"https://api.github.com/repos/palantir/atlasdb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/palantir/atlasdb/pulls/2176/commits"},"statuses":{"href":"https://api.github.com/repos/palantir/atlasdb/statuses/f2557c54b6108b92958a6423a088a837a64dd96a"}}}}
{"actor":{"display_login":"nziebart"}}