{"action":"created","comment":{"url":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/comments/98764821","pull_request_review_id":19417274,"id":98764821,"diff_hunk":"@@ -0,0 +1,71 @@\n+#!/usr/bin/env ruby\n+\n+# This script runs the AWS Data Migration Service tasks that replicate data from\n+# our reporting RDS database to our Redshift database. It is highly tied to the\n+# AWS DMS tasks (https://console.aws.amazon.com/dms/home?region=us-east-1#tasks:)\n+# and the CDO.dms_tasks secret.\n+\n+require 'cdo/hip_chat'\n+\n+# The list of DMS tasks to start, as well as the length of time that should\n+# elapse before starting the next task (an overestimate of completion time).\n+# The hash keys are expected to be keys in CDO.dms_tasks through which the task\n+# ARNs are retrieved.\n+TASKS = {\n+  oneoff_survey_hierarchy: {\n+    duration: 5\n+  },\n+  oneoff_user_progress_hierarchy: {\n+    duration: 640\n+  },\n+  oneoff_teaching_hierarchy: {\n+    duration: 10\n+  },\n+  oneoff_script_hierarchy: {\n+    duration: 5\n+  },\n+  oneoff_user_hierarchy_pii: {\n+    duration: 90\n+  },\n+  oneoff_plc_hierarchy: {\n+    duration: 10\n+  },\n+  oneoff_user_proficiency_hierarchy: {\n+    duration: 15\n+  },\n+  oneoff_contacts_hierarchy_pii: {\n+    duration: 10\n+  },\n+  oneoff_multi_hierarchy: {\n+    duration: 10\n+  },\n+  oneoff_metrics_hierarchy: {\n+    duration: 10\n+  },\n+  oneoff_user_hierarchy: {\n+    duration: 40\n+  },\n+  oneoff_hint_hierarchy: {\n+    duration: 30\n+  }\n+}.freeze\n+\n+TASK_ARNS = CDO.dms_tasks.freeze\n+\n+TASKS.each do |task_name, task_info|\n+  HipChat.message 'infra-dms',\n+    \"Starting DMS task #{task_name} (sleep duration: #{task_info[:duration]})...\",\n+    color: green\n+\n+  task_arn = TASK_ARNS[task_name]\n+  if task_arn\n+    system \"aws dms start-replication-task --replication-task-arn arn:#{task_arn} --start-replication-task-type reload-target\"\n+    sleep task_info[:duration]","path":"bin/cron/dms_tasks","position":null,"original_position":63,"commit_id":"9ab99cdd244cd079688357cc57a9334c4aaea6b5","original_commit_id":"9c46b7d0dbb8ffd80763d175e63b0b4927401ccc","user":{"login":"ashercodeorg","id":14206260,"avatar_url":"https://avatars.githubusercontent.com/u/14206260?v=3","gravatar_id":"","url":"https://api.github.com/users/ashercodeorg","html_url":"https://github.com/ashercodeorg","followers_url":"https://api.github.com/users/ashercodeorg/followers","following_url":"https://api.github.com/users/ashercodeorg/following{/other_user}","gists_url":"https://api.github.com/users/ashercodeorg/gists{/gist_id}","starred_url":"https://api.github.com/users/ashercodeorg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ashercodeorg/subscriptions","organizations_url":"https://api.github.com/users/ashercodeorg/orgs","repos_url":"https://api.github.com/users/ashercodeorg/repos","events_url":"https://api.github.com/users/ashercodeorg/events{/privacy}","received_events_url":"https://api.github.com/users/ashercodeorg/received_events","type":"User","site_admin":false},"body":"The DMS documentation suggests that running too many tasks (relative to the replication instance size, RDS DB size, Redshift DB size, etc.) can cause issues. And we've certainly observed this empirically.","created_at":"2017-01-31T20:39:16Z","updated_at":"2017-01-31T20:39:16Z","html_url":"https://github.com/code-dot-org/code-dot-org/pull/12940#discussion_r98764821","pull_request_url":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/12940","_links":{"self":{"href":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/comments/98764821"},"html":{"href":"https://github.com/code-dot-org/code-dot-org/pull/12940#discussion_r98764821"},"pull_request":{"href":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/12940"}}},"pull_request":{"url":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/12940","id":103802544,"html_url":"https://github.com/code-dot-org/code-dot-org/pull/12940","diff_url":"https://github.com/code-dot-org/code-dot-org/pull/12940.diff","patch_url":"https://github.com/code-dot-org/code-dot-org/pull/12940.patch","issue_url":"https://api.github.com/repos/code-dot-org/code-dot-org/issues/12940","number":12940,"state":"open","locked":false,"title":"Creates cron job to start DMS replication tasks.","user":{"login":"ashercodeorg","id":14206260,"avatar_url":"https://avatars.githubusercontent.com/u/14206260?v=3","gravatar_id":"","url":"https://api.github.com/users/ashercodeorg","html_url":"https://github.com/ashercodeorg","followers_url":"https://api.github.com/users/ashercodeorg/followers","following_url":"https://api.github.com/users/ashercodeorg/following{/other_user}","gists_url":"https://api.github.com/users/ashercodeorg/gists{/gist_id}","starred_url":"https://api.github.com/users/ashercodeorg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ashercodeorg/subscriptions","organizations_url":"https://api.github.com/users/ashercodeorg/orgs","repos_url":"https://api.github.com/users/ashercodeorg/repos","events_url":"https://api.github.com/users/ashercodeorg/events{/privacy}","received_events_url":"https://api.github.com/users/ashercodeorg/received_events","type":"User","site_admin":false},"body":"Note that, since `CDO.dms_tasks` hasn't propagated to production, this code is mostly untested. My plan is to merge, run (confirm working) manually on production, then cron the job.","created_at":"2017-01-30T19:26:18Z","updated_at":"2017-01-31T20:39:16Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7838fbd318ae252cd0c5b21b1bfeeb0e89c270fc","assignee":{"login":"wjordan","id":56541,"avatar_url":"https://avatars.githubusercontent.com/u/56541?v=3","gravatar_id":"","url":"https://api.github.com/users/wjordan","html_url":"https://github.com/wjordan","followers_url":"https://api.github.com/users/wjordan/followers","following_url":"https://api.github.com/users/wjordan/following{/other_user}","gists_url":"https://api.github.com/users/wjordan/gists{/gist_id}","starred_url":"https://api.github.com/users/wjordan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wjordan/subscriptions","organizations_url":"https://api.github.com/users/wjordan/orgs","repos_url":"https://api.github.com/users/wjordan/repos","events_url":"https://api.github.com/users/wjordan/events{/privacy}","received_events_url":"https://api.github.com/users/wjordan/received_events","type":"User","site_admin":false},"assignees":[{"login":"wjordan","id":56541,"avatar_url":"https://avatars.githubusercontent.com/u/56541?v=3","gravatar_id":"","url":"https://api.github.com/users/wjordan","html_url":"https://github.com/wjordan","followers_url":"https://api.github.com/users/wjordan/followers","following_url":"https://api.github.com/users/wjordan/following{/other_user}","gists_url":"https://api.github.com/users/wjordan/gists{/gist_id}","starred_url":"https://api.github.com/users/wjordan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wjordan/subscriptions","organizations_url":"https://api.github.com/users/wjordan/orgs","repos_url":"https://api.github.com/users/wjordan/repos","events_url":"https://api.github.com/users/wjordan/events{/privacy}","received_events_url":"https://api.github.com/users/wjordan/received_events","type":"User","site_admin":false},{"login":"ewjordan","id":70630,"avatar_url":"https://avatars.githubusercontent.com/u/70630?v=3","gravatar_id":"","url":"https://api.github.com/users/ewjordan","html_url":"https://github.com/ewjordan","followers_url":"https://api.github.com/users/ewjordan/followers","following_url":"https://api.github.com/users/ewjordan/following{/other_user}","gists_url":"https://api.github.com/users/ewjordan/gists{/gist_id}","starred_url":"https://api.github.com/users/ewjordan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ewjordan/subscriptions","organizations_url":"https://api.github.com/users/ewjordan/orgs","repos_url":"https://api.github.com/users/ewjordan/repos","events_url":"https://api.github.com/users/ewjordan/events{/privacy}","received_events_url":"https://api.github.com/users/ewjordan/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/12940/commits","review_comments_url":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/12940/comments","review_comment_url":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/comments{/number}","comments_url":"https://api.github.com/repos/code-dot-org/code-dot-org/issues/12940/comments","statuses_url":"https://api.github.com/repos/code-dot-org/code-dot-org/statuses/9ab99cdd244cd079688357cc57a9334c4aaea6b5","head":{"label":"code-dot-org:cronDmsTasks","ref":"cronDmsTasks","sha":"9ab99cdd244cd079688357cc57a9334c4aaea6b5","user":{"login":"code-dot-org","id":5124627,"avatar_url":"https://avatars.githubusercontent.com/u/5124627?v=3","gravatar_id":"","url":"https://api.github.com/users/code-dot-org","html_url":"https://github.com/code-dot-org","followers_url":"https://api.github.com/users/code-dot-org/followers","following_url":"https://api.github.com/users/code-dot-org/following{/other_user}","gists_url":"https://api.github.com/users/code-dot-org/gists{/gist_id}","starred_url":"https://api.github.com/users/code-dot-org/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/code-dot-org/subscriptions","organizations_url":"https://api.github.com/users/code-dot-org/orgs","repos_url":"https://api.github.com/users/code-dot-org/repos","events_url":"https://api.github.com/users/code-dot-org/events{/privacy}","received_events_url":"https://api.github.com/users/code-dot-org/received_events","type":"Organization","site_admin":false},"repo":{"id":24250009,"name":"code-dot-org","full_name":"code-dot-org/code-dot-org","owner":{"login":"code-dot-org","id":5124627,"avatar_url":"https://avatars.githubusercontent.com/u/5124627?v=3","gravatar_id":"","url":"https://api.github.com/users/code-dot-org","html_url":"https://github.com/code-dot-org","followers_url":"https://api.github.com/users/code-dot-org/followers","following_url":"https://api.github.com/users/code-dot-org/following{/other_user}","gists_url":"https://api.github.com/users/code-dot-org/gists{/gist_id}","starred_url":"https://api.github.com/users/code-dot-org/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/code-dot-org/subscriptions","organizations_url":"https://api.github.com/users/code-dot-org/orgs","repos_url":"https://api.github.com/users/code-dot-org/repos","events_url":"https://api.github.com/users/code-dot-org/events{/privacy}","received_events_url":"https://api.github.com/users/code-dot-org/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/code-dot-org/code-dot-org","description":"The code powering code.org and studio.code.org","fork":false,"url":"https://api.github.com/repos/code-dot-org/code-dot-org","forks_url":"https://api.github.com/repos/code-dot-org/code-dot-org/forks","keys_url":"https://api.github.com/repos/code-dot-org/code-dot-org/keys{/key_id}","collaborators_url":"https://api.github.com/repos/code-dot-org/code-dot-org/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/code-dot-org/code-dot-org/teams","hooks_url":"https://api.github.com/repos/code-dot-org/code-dot-org/hooks","issue_events_url":"https://api.github.com/repos/code-dot-org/code-dot-org/issues/events{/number}","events_url":"https://api.github.com/repos/code-dot-org/code-dot-org/events","assignees_url":"https://api.github.com/repos/code-dot-org/code-dot-org/assignees{/user}","branches_url":"https://api.github.com/repos/code-dot-org/code-dot-org/branches{/branch}","tags_url":"https://api.github.com/repos/code-dot-org/code-dot-org/tags","blobs_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/refs{/sha}","trees_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/trees{/sha}","statuses_url":"https://api.github.com/repos/code-dot-org/code-dot-org/statuses/{sha}","languages_url":"https://api.github.com/repos/code-dot-org/code-dot-org/languages","stargazers_url":"https://api.github.com/repos/code-dot-org/code-dot-org/stargazers","contributors_url":"https://api.github.com/repos/code-dot-org/code-dot-org/contributors","subscribers_url":"https://api.github.com/repos/code-dot-org/code-dot-org/subscribers","subscription_url":"https://api.github.com/repos/code-dot-org/code-dot-org/subscription","commits_url":"https://api.github.com/repos/code-dot-org/code-dot-org/commits{/sha}","git_commits_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/commits{/sha}","comments_url":"https://api.github.com/repos/code-dot-org/code-dot-org/comments{/number}","issue_comment_url":"https://api.github.com/repos/code-dot-org/code-dot-org/issues/comments{/number}","contents_url":"https://api.github.com/repos/code-dot-org/code-dot-org/contents/{+path}","compare_url":"https://api.github.com/repos/code-dot-org/code-dot-org/compare/{base}...{head}","merges_url":"https://api.github.com/repos/code-dot-org/code-dot-org/merges","archive_url":"https://api.github.com/repos/code-dot-org/code-dot-org/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/code-dot-org/code-dot-org/downloads","issues_url":"https://api.github.com/repos/code-dot-org/code-dot-org/issues{/number}","pulls_url":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls{/number}","milestones_url":"https://api.github.com/repos/code-dot-org/code-dot-org/milestones{/number}","notifications_url":"https://api.github.com/repos/code-dot-org/code-dot-org/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/code-dot-org/code-dot-org/labels{/name}","releases_url":"https://api.github.com/repos/code-dot-org/code-dot-org/releases{/id}","deployments_url":"https://api.github.com/repos/code-dot-org/code-dot-org/deployments","created_at":"2014-09-20T00:46:11Z","updated_at":"2017-01-28T22:46:26Z","pushed_at":"2017-01-31T20:38:58Z","git_url":"git://github.com/code-dot-org/code-dot-org.git","ssh_url":"git@github.com:code-dot-org/code-dot-org.git","clone_url":"https://github.com/code-dot-org/code-dot-org.git","svn_url":"https://github.com/code-dot-org/code-dot-org","homepage":"http://code.org","size":5219209,"stargazers_count":160,"watchers_count":160,"language":"JavaScript","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":105,"mirror_url":null,"open_issues_count":87,"forks":105,"open_issues":87,"watchers":160,"default_branch":"staging"}},"base":{"label":"code-dot-org:staging","ref":"staging","sha":"10f2d8b865fd2abac3df69220ac7a60abb11c8e8","user":{"login":"code-dot-org","id":5124627,"avatar_url":"https://avatars.githubusercontent.com/u/5124627?v=3","gravatar_id":"","url":"https://api.github.com/users/code-dot-org","html_url":"https://github.com/code-dot-org","followers_url":"https://api.github.com/users/code-dot-org/followers","following_url":"https://api.github.com/users/code-dot-org/following{/other_user}","gists_url":"https://api.github.com/users/code-dot-org/gists{/gist_id}","starred_url":"https://api.github.com/users/code-dot-org/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/code-dot-org/subscriptions","organizations_url":"https://api.github.com/users/code-dot-org/orgs","repos_url":"https://api.github.com/users/code-dot-org/repos","events_url":"https://api.github.com/users/code-dot-org/events{/privacy}","received_events_url":"https://api.github.com/users/code-dot-org/received_events","type":"Organization","site_admin":false},"repo":{"id":24250009,"name":"code-dot-org","full_name":"code-dot-org/code-dot-org","owner":{"login":"code-dot-org","id":5124627,"avatar_url":"https://avatars.githubusercontent.com/u/5124627?v=3","gravatar_id":"","url":"https://api.github.com/users/code-dot-org","html_url":"https://github.com/code-dot-org","followers_url":"https://api.github.com/users/code-dot-org/followers","following_url":"https://api.github.com/users/code-dot-org/following{/other_user}","gists_url":"https://api.github.com/users/code-dot-org/gists{/gist_id}","starred_url":"https://api.github.com/users/code-dot-org/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/code-dot-org/subscriptions","organizations_url":"https://api.github.com/users/code-dot-org/orgs","repos_url":"https://api.github.com/users/code-dot-org/repos","events_url":"https://api.github.com/users/code-dot-org/events{/privacy}","received_events_url":"https://api.github.com/users/code-dot-org/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/code-dot-org/code-dot-org","description":"The code powering code.org and studio.code.org","fork":false,"url":"https://api.github.com/repos/code-dot-org/code-dot-org","forks_url":"https://api.github.com/repos/code-dot-org/code-dot-org/forks","keys_url":"https://api.github.com/repos/code-dot-org/code-dot-org/keys{/key_id}","collaborators_url":"https://api.github.com/repos/code-dot-org/code-dot-org/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/code-dot-org/code-dot-org/teams","hooks_url":"https://api.github.com/repos/code-dot-org/code-dot-org/hooks","issue_events_url":"https://api.github.com/repos/code-dot-org/code-dot-org/issues/events{/number}","events_url":"https://api.github.com/repos/code-dot-org/code-dot-org/events","assignees_url":"https://api.github.com/repos/code-dot-org/code-dot-org/assignees{/user}","branches_url":"https://api.github.com/repos/code-dot-org/code-dot-org/branches{/branch}","tags_url":"https://api.github.com/repos/code-dot-org/code-dot-org/tags","blobs_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/refs{/sha}","trees_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/trees{/sha}","statuses_url":"https://api.github.com/repos/code-dot-org/code-dot-org/statuses/{sha}","languages_url":"https://api.github.com/repos/code-dot-org/code-dot-org/languages","stargazers_url":"https://api.github.com/repos/code-dot-org/code-dot-org/stargazers","contributors_url":"https://api.github.com/repos/code-dot-org/code-dot-org/contributors","subscribers_url":"https://api.github.com/repos/code-dot-org/code-dot-org/subscribers","subscription_url":"https://api.github.com/repos/code-dot-org/code-dot-org/subscription","commits_url":"https://api.github.com/repos/code-dot-org/code-dot-org/commits{/sha}","git_commits_url":"https://api.github.com/repos/code-dot-org/code-dot-org/git/commits{/sha}","comments_url":"https://api.github.com/repos/code-dot-org/code-dot-org/comments{/number}","issue_comment_url":"https://api.github.com/repos/code-dot-org/code-dot-org/issues/comments{/number}","contents_url":"https://api.github.com/repos/code-dot-org/code-dot-org/contents/{+path}","compare_url":"https://api.github.com/repos/code-dot-org/code-dot-org/compare/{base}...{head}","merges_url":"https://api.github.com/repos/code-dot-org/code-dot-org/merges","archive_url":"https://api.github.com/repos/code-dot-org/code-dot-org/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/code-dot-org/code-dot-org/downloads","issues_url":"https://api.github.com/repos/code-dot-org/code-dot-org/issues{/number}","pulls_url":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls{/number}","milestones_url":"https://api.github.com/repos/code-dot-org/code-dot-org/milestones{/number}","notifications_url":"https://api.github.com/repos/code-dot-org/code-dot-org/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/code-dot-org/code-dot-org/labels{/name}","releases_url":"https://api.github.com/repos/code-dot-org/code-dot-org/releases{/id}","deployments_url":"https://api.github.com/repos/code-dot-org/code-dot-org/deployments","created_at":"2014-09-20T00:46:11Z","updated_at":"2017-01-28T22:46:26Z","pushed_at":"2017-01-31T20:38:58Z","git_url":"git://github.com/code-dot-org/code-dot-org.git","ssh_url":"git@github.com:code-dot-org/code-dot-org.git","clone_url":"https://github.com/code-dot-org/code-dot-org.git","svn_url":"https://github.com/code-dot-org/code-dot-org","homepage":"http://code.org","size":5219209,"stargazers_count":160,"watchers_count":160,"language":"JavaScript","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":105,"mirror_url":null,"open_issues_count":87,"forks":105,"open_issues":87,"watchers":160,"default_branch":"staging"}},"_links":{"self":{"href":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/12940"},"html":{"href":"https://github.com/code-dot-org/code-dot-org/pull/12940"},"issue":{"href":"https://api.github.com/repos/code-dot-org/code-dot-org/issues/12940"},"comments":{"href":"https://api.github.com/repos/code-dot-org/code-dot-org/issues/12940/comments"},"review_comments":{"href":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/12940/comments"},"review_comment":{"href":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/code-dot-org/code-dot-org/pulls/12940/commits"},"statuses":{"href":"https://api.github.com/repos/code-dot-org/code-dot-org/statuses/9ab99cdd244cd079688357cc57a9334c4aaea6b5"}}}}
{"actor":{"display_login":"ashercodeorg"}}