{"action":"created","comment":{"url":"https://api.github.com/repos/awslabs/sockeye/pulls/comments/156399046","pull_request_review_id":82869301,"id":156399046,"diff_hunk":"@@ -100,36 +105,569 @@ def get_bucket(seq_len: int, buckets: List[int]) -> Optional[int]:\n     return buckets[bucket_idx]\n \n \n-def length_statistics(source_sentences: Iterable[List[Any]],\n-                      target_sentences: Iterable[List[Any]]) -> Tuple[float, float]:\n+class BucketBatchSize:\n+    \"\"\"\n+    :param bucket: The corresponding bucket.\n+    :param batch_size: Number of sequences in each batch.\n+    :param average_words_per_batch: Approximate number of non-padding tokens in each batch.\n+    \"\"\"\n+\n+    def __init__(self, bucket: Tuple[int, int], batch_size: int, average_words_per_batch: float) -> None:\n+        self.bucket = bucket\n+        self.batch_size = batch_size\n+        self.average_words_per_batch = average_words_per_batch\n+\n+\n+def define_bucket_batch_sizes(buckets: List[Tuple[int, int]],\n+                              batch_size: int,\n+                              batch_by_words: bool,\n+                              batch_num_devices: int,\n+                              data_target_average_len: List[Optional[float]]) -> List[BucketBatchSize]:\n+    \"\"\"\n+    Computes bucket-specific batch sizes (sentences, average_words).\n+\n+    If sentence-based batching: number of sentences is the same for each batch, determines the\n+    number of words. Hence all batch sizes for each bucket are equal.\n+\n+    If word-based batching: number of sentences for each batch is set to the multiple of number\n+    of devices that produces the number of words closest to the target batch size.  Average\n+    target sentence length (non-padding symbols) is used for word number calculations.\n+\n+    :param buckets: Bucket list.\n+    :param batch_size: Batch size.\n+    :param batch_by_words: Batch by words.\n+    :param batch_num_devices: Number of devices.\n+    :param data_target_average_len: Optional average target length for each bucket.\n+    \"\"\"\n+    check_condition(len(data_target_average_len) == len(buckets),\n+                    \"Must provide None or average target length for each bucket\")\n+    data_target_average_len = list(data_target_average_len)\n+    bucket_batch_sizes = []  # type: List[BucketBatchSize]\n+    largest_total_num_words = 0\n+    for buck_idx, bucket in enumerate(buckets):\n+        # Target/label length with padding\n+        padded_seq_len = bucket[1]\n+        # Average target/label length excluding padding\n+        if data_target_average_len[buck_idx] is None:\n+            data_target_average_len[buck_idx] = padded_seq_len\n+        average_seq_len = data_target_average_len[buck_idx]\n+\n+        # Word-based: num words determines num sentences\n+        # Sentence-based: num sentences determines num words\n+        if batch_by_words:\n+            check_condition(padded_seq_len <= batch_size, \"Word batch size must cover sequence lengths for all\"\n+                                                          \" buckets: (%d > %d)\" % (padded_seq_len, batch_size))\n+            # Multiple of number of devices (int) closest to target number of words, assuming each sentence is of\n+            # average length\n+            batch_size_seq = batch_num_devices * round((batch_size / average_seq_len) / batch_num_devices)\n+            batch_size_word = batch_size_seq * average_seq_len\n+        else:\n+            batch_size_seq = batch_size\n+            batch_size_word = batch_size_seq * average_seq_len\n+        bucket_batch_sizes.append(BucketBatchSize(bucket, batch_size_seq, batch_size_word))\n+        # Track largest number of word samples in a batch\n+        largest_total_num_words = max(largest_total_num_words, batch_size_seq * max(*bucket))\n+\n+    # Final step: guarantee that largest bucket by sequence length also has largest total batch size.\n+    # When batching by sentences, this will already be the case.\n+    if batch_by_words:\n+        padded_seq_len = max(*buckets[-1])\n+        average_seq_len = data_target_average_len[-1]\n+        while bucket_batch_sizes[-1].batch_size * padded_seq_len < largest_total_num_words:\n+            bucket_batch_sizes[-1] = BucketBatchSize(\n+                bucket_batch_sizes[-1].bucket,\n+                bucket_batch_sizes[-1].batch_size + batch_num_devices,\n+                bucket_batch_sizes[-1].average_words_per_batch + batch_num_devices * average_seq_len)\n+    return bucket_batch_sizes\n+\n+\n+def calculate_length_statistics(source_sentences: Iterable[List[Any]],\n+                                target_sentences: Iterable[List[Any]],\n+                                max_seq_len_source: int,\n+                                max_seq_len_target: int) -> 'LengthStatistics':\n     \"\"\"\n     Returns mean and standard deviation of target-to-source length ratios of parallel corpus.\n \n     :param source_sentences: Source sentences.\n     :param target_sentences: Target sentences.\n-    :return: Mean and standard deviation of length ratios.\n+    :param max_seq_len_source: Maximum source sequence length.\n+    :param max_seq_len_target: Maximum target sequence length.\n+    :return: The number of sentences as well as the mean and standard deviation of target to source length ratios.\n     \"\"\"\n-    length_ratios = np.array([len(t)/float(len(s)) for t, s in zip(target_sentences, source_sentences)])\n-    mean = np.asscalar(np.mean(length_ratios))\n-    std = np.asscalar(np.std(length_ratios))\n-    return mean, std\n+    mean_and_variance = OnlineMeanAndVariance()\n+\n+    for target, source in zip(target_sentences, source_sentences):\n+        source_len = len(source)\n+        target_len = len(target)\n+        if source_len > max_seq_len_source or target_len > max_seq_len_target:\n+            continue\n+\n+        length_ratio = target_len / source_len\n+        mean_and_variance.update(length_ratio)\n+\n+    num_sents = mean_and_variance.count\n+    mean = mean_and_variance.mean\n+    std = math.sqrt(mean_and_variance.variance)\n+    return LengthStatistics(num_sents, mean, std)\n+\n+\n+def analyze_sequence_lengths(source: str,\n+                             target: str,\n+                             vocab_source: Dict[str, int],\n+                             vocab_target: Dict[str, int],\n+                             max_seq_len_source: int,\n+                             max_seq_len_target: int) -> 'LengthStatistics':\n+    train_source_sentences = SequenceReader(source, vocab_source, add_bos=False)\n+    # Length statistics are calculated on the raw sentences without special tokens, such as the BOS, as these can\n+    # have a a large impact on the length ratios, especially with lots of short sequences.\n+    train_target_sentences = SequenceReader(target, vocab_target, add_bos=False)\n+\n+    length_statistics = calculate_length_statistics(train_source_sentences, train_target_sentences,\n+                                                    max_seq_len_source,\n+                                                    # Take into account the BOS symbol that is added later\n+                                                    max_seq_len_target - 1)\n+    check_condition(train_source_sentences.is_done() and train_target_sentences.is_done(),\n+                    \"Different number of lines in source and target data.\")\n+\n+    logger.info(\"%d sequences of maximum length (%d, %d) in '%s' and '%s'.\",\n+                length_statistics.num_sents, max_seq_len_source, max_seq_len_target, source, target)\n+    logger.info(\"Mean training target/source length ratio: %.2f (+-%.2f)\",\n+                length_statistics.length_ratio_mean,\n+                length_statistics.length_ratio_std)\n+    return length_statistics\n+\n+\n+class DataStatisticsAccumulator:\n+\n+    def __init__(self,\n+                 buckets: List[Tuple[int, int]],\n+                 vocab_source: Dict[str, int],\n+                 vocab_target: Dict[str, int],\n+                 length_ratio_mean: float,\n+                 length_ratio_std: float) -> None:\n+        self.buckets = buckets\n+        num_buckets = len(buckets)\n+        self.length_ratio_mean = length_ratio_mean\n+        self.length_ratio_std = length_ratio_std\n+        self.unk_id_source = vocab_source[C.UNK_SYMBOL]\n+        self.unk_id_target = vocab_target[C.UNK_SYMBOL]\n+        self.size_vocab_source = len(vocab_source)\n+        self.size_vocab_target = len(vocab_target)\n+        self.num_sents = 0\n+        self.num_discarded = 0\n+        self.num_tokens_source = 0\n+        self.num_tokens_target = 0\n+        self.num_unks_source = 0\n+        self.num_unks_target = 0\n+        self.max_observed_len_source = 0\n+        self.max_observed_len_target = 0\n+        self._mean_len_target_per_bucket = [OnlineMeanAndVariance() for _ in range(num_buckets)]\n+\n+    def sequence_pair(self,\n+                      source: List[int],\n+                      target: List[int],\n+                      bucket_idx: Optional[int]):\n+        source_len = len(source)\n+        target_len = len(target)\n+\n+        if bucket_idx is None:\n+            self.num_discarded += 1\n+            return\n+\n+        self._mean_len_target_per_bucket[bucket_idx].update(target_len)\n+\n+        self.num_sents += 1\n+        self.num_tokens_source += source_len\n+        self.num_tokens_target += target_len\n+        if source_len > self.max_observed_len_source:","path":"sockeye/data_io.py","position":235,"original_position":235,"commit_id":"3f2742992648f282db6ed9b4947c31c6774ffc1d","original_commit_id":"026a82b3ce4d3d2aac7485cb1b4d58bfdc5568a8","user":{"login":"tdomhan","id":363737,"avatar_url":"https://avatars1.githubusercontent.com/u/363737?v=4","gravatar_id":"","url":"https://api.github.com/users/tdomhan","html_url":"https://github.com/tdomhan","followers_url":"https://api.github.com/users/tdomhan/followers","following_url":"https://api.github.com/users/tdomhan/following{/other_user}","gists_url":"https://api.github.com/users/tdomhan/gists{/gist_id}","starred_url":"https://api.github.com/users/tdomhan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tdomhan/subscriptions","organizations_url":"https://api.github.com/users/tdomhan/orgs","repos_url":"https://api.github.com/users/tdomhan/repos","events_url":"https://api.github.com/users/tdomhan/events{/privacy}","received_events_url":"https://api.github.com/users/tdomhan/received_events","type":"User","site_admin":false},"body":"sure","created_at":"2017-12-12T15:27:55Z","updated_at":"2017-12-12T15:27:55Z","html_url":"https://github.com/awslabs/sockeye/pull/241#discussion_r156399046","pull_request_url":"https://api.github.com/repos/awslabs/sockeye/pulls/241","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/comments/156399046"},"html":{"href":"https://github.com/awslabs/sockeye/pull/241#discussion_r156399046"},"pull_request":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/241"}},"in_reply_to_id":156386934},"pull_request":{"url":"https://api.github.com/repos/awslabs/sockeye/pulls/241","id":157813692,"html_url":"https://github.com/awslabs/sockeye/pull/241","diff_url":"https://github.com/awslabs/sockeye/pull/241.diff","patch_url":"https://github.com/awslabs/sockeye/pull/241.patch","issue_url":"https://api.github.com/repos/awslabs/sockeye/issues/241","number":241,"state":"open","locked":false,"title":"Sharded data iterator.","user":{"login":"tdomhan","id":363737,"avatar_url":"https://avatars1.githubusercontent.com/u/363737?v=4","gravatar_id":"","url":"https://api.github.com/users/tdomhan","html_url":"https://github.com/tdomhan","followers_url":"https://api.github.com/users/tdomhan/followers","following_url":"https://api.github.com/users/tdomhan/following{/other_user}","gists_url":"https://api.github.com/users/tdomhan/gists{/gist_id}","starred_url":"https://api.github.com/users/tdomhan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tdomhan/subscriptions","organizations_url":"https://api.github.com/users/tdomhan/orgs","repos_url":"https://api.github.com/users/tdomhan/repos","events_url":"https://api.github.com/users/tdomhan/events{/privacy}","received_events_url":"https://api.github.com/users/tdomhan/received_events","type":"User","site_admin":false},"body":"Data can now optionally be sharded and preprocessed once using the new CLI  `sockeye.prepare_data`. Instead of using the ```--source``` and ```--target``` arguments ```sockeye.train``` now accepts a ```--prepared-data``` argument pointing to the folder containing the preprocessed and sharded data. Using the raw training data is still possible and now consumes less memory.\r\n\r\n\r\n#### Pull Request Checklist ##\r\n- [X] Changes are complete (if posting work-in-progress code, prefix your pull request title with '[WIP]'\r\nuntil you can check this box.\r\n- [X] Unit tests pass (`pytest`)\r\n- [ ] System tests pass (`pytest test/system`)\r\n- [X] Passed code style checking (`./pre-commit.sh` or manual run of pylint & mypy)\r\n- [X] Updated major/minor version in `sockeye/__init__.py`. Major version bump if this is a backwards incompatible change.\r\n- [X] Updated CHANGELOG.md\r\n","created_at":"2017-12-12T12:20:46Z","updated_at":"2017-12-12T15:27:55Z","closed_at":null,"merged_at":null,"merge_commit_sha":"774015a57b83182c8dd32db3018f5e1b697a5480","assignee":null,"assignees":[],"requested_reviewers":[{"login":"mjdenkowski","id":636653,"avatar_url":"https://avatars3.githubusercontent.com/u/636653?v=4","gravatar_id":"","url":"https://api.github.com/users/mjdenkowski","html_url":"https://github.com/mjdenkowski","followers_url":"https://api.github.com/users/mjdenkowski/followers","following_url":"https://api.github.com/users/mjdenkowski/following{/other_user}","gists_url":"https://api.github.com/users/mjdenkowski/gists{/gist_id}","starred_url":"https://api.github.com/users/mjdenkowski/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mjdenkowski/subscriptions","organizations_url":"https://api.github.com/users/mjdenkowski/orgs","repos_url":"https://api.github.com/users/mjdenkowski/repos","events_url":"https://api.github.com/users/mjdenkowski/events{/privacy}","received_events_url":"https://api.github.com/users/mjdenkowski/received_events","type":"User","site_admin":false},{"login":"davvil","id":683589,"avatar_url":"https://avatars0.githubusercontent.com/u/683589?v=4","gravatar_id":"","url":"https://api.github.com/users/davvil","html_url":"https://github.com/davvil","followers_url":"https://api.github.com/users/davvil/followers","following_url":"https://api.github.com/users/davvil/following{/other_user}","gists_url":"https://api.github.com/users/davvil/gists{/gist_id}","starred_url":"https://api.github.com/users/davvil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davvil/subscriptions","organizations_url":"https://api.github.com/users/davvil/orgs","repos_url":"https://api.github.com/users/davvil/repos","events_url":"https://api.github.com/users/davvil/events{/privacy}","received_events_url":"https://api.github.com/users/davvil/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/awslabs/sockeye/pulls/241/commits","review_comments_url":"https://api.github.com/repos/awslabs/sockeye/pulls/241/comments","review_comment_url":"https://api.github.com/repos/awslabs/sockeye/pulls/comments{/number}","comments_url":"https://api.github.com/repos/awslabs/sockeye/issues/241/comments","statuses_url":"https://api.github.com/repos/awslabs/sockeye/statuses/3f2742992648f282db6ed9b4947c31c6774ffc1d","head":{"label":"awslabs:sharded_iter","ref":"sharded_iter","sha":"3f2742992648f282db6ed9b4947c31c6774ffc1d","user":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars0.githubusercontent.com/u/3299148?v=4","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"repo":{"id":93721432,"name":"sockeye","full_name":"awslabs/sockeye","owner":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars0.githubusercontent.com/u/3299148?v=4","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/awslabs/sockeye","description":"Sequence-to-sequence framework with a focus on Neural Machine Translation based on Apache MXNet","fork":false,"url":"https://api.github.com/repos/awslabs/sockeye","forks_url":"https://api.github.com/repos/awslabs/sockeye/forks","keys_url":"https://api.github.com/repos/awslabs/sockeye/keys{/key_id}","collaborators_url":"https://api.github.com/repos/awslabs/sockeye/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/awslabs/sockeye/teams","hooks_url":"https://api.github.com/repos/awslabs/sockeye/hooks","issue_events_url":"https://api.github.com/repos/awslabs/sockeye/issues/events{/number}","events_url":"https://api.github.com/repos/awslabs/sockeye/events","assignees_url":"https://api.github.com/repos/awslabs/sockeye/assignees{/user}","branches_url":"https://api.github.com/repos/awslabs/sockeye/branches{/branch}","tags_url":"https://api.github.com/repos/awslabs/sockeye/tags","blobs_url":"https://api.github.com/repos/awslabs/sockeye/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/awslabs/sockeye/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/awslabs/sockeye/git/refs{/sha}","trees_url":"https://api.github.com/repos/awslabs/sockeye/git/trees{/sha}","statuses_url":"https://api.github.com/repos/awslabs/sockeye/statuses/{sha}","languages_url":"https://api.github.com/repos/awslabs/sockeye/languages","stargazers_url":"https://api.github.com/repos/awslabs/sockeye/stargazers","contributors_url":"https://api.github.com/repos/awslabs/sockeye/contributors","subscribers_url":"https://api.github.com/repos/awslabs/sockeye/subscribers","subscription_url":"https://api.github.com/repos/awslabs/sockeye/subscription","commits_url":"https://api.github.com/repos/awslabs/sockeye/commits{/sha}","git_commits_url":"https://api.github.com/repos/awslabs/sockeye/git/commits{/sha}","comments_url":"https://api.github.com/repos/awslabs/sockeye/comments{/number}","issue_comment_url":"https://api.github.com/repos/awslabs/sockeye/issues/comments{/number}","contents_url":"https://api.github.com/repos/awslabs/sockeye/contents/{+path}","compare_url":"https://api.github.com/repos/awslabs/sockeye/compare/{base}...{head}","merges_url":"https://api.github.com/repos/awslabs/sockeye/merges","archive_url":"https://api.github.com/repos/awslabs/sockeye/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/awslabs/sockeye/downloads","issues_url":"https://api.github.com/repos/awslabs/sockeye/issues{/number}","pulls_url":"https://api.github.com/repos/awslabs/sockeye/pulls{/number}","milestones_url":"https://api.github.com/repos/awslabs/sockeye/milestones{/number}","notifications_url":"https://api.github.com/repos/awslabs/sockeye/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/awslabs/sockeye/labels{/name}","releases_url":"https://api.github.com/repos/awslabs/sockeye/releases{/id}","deployments_url":"https://api.github.com/repos/awslabs/sockeye/deployments","created_at":"2017-06-08T07:44:30Z","updated_at":"2017-12-11T15:31:08Z","pushed_at":"2017-12-12T15:25:54Z","git_url":"git://github.com/awslabs/sockeye.git","ssh_url":"git@github.com:awslabs/sockeye.git","clone_url":"https://github.com/awslabs/sockeye.git","svn_url":"https://github.com/awslabs/sockeye","homepage":"","size":1433,"stargazers_count":288,"watchers_count":288,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":79,"mirror_url":null,"archived":false,"open_issues_count":7,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":79,"open_issues":7,"watchers":288,"default_branch":"master"}},"base":{"label":"awslabs:master","ref":"master","sha":"bd4a23609efdcf0d0c52fcacc8438020b5f6a7f2","user":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars0.githubusercontent.com/u/3299148?v=4","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"repo":{"id":93721432,"name":"sockeye","full_name":"awslabs/sockeye","owner":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars0.githubusercontent.com/u/3299148?v=4","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/awslabs/sockeye","description":"Sequence-to-sequence framework with a focus on Neural Machine Translation based on Apache MXNet","fork":false,"url":"https://api.github.com/repos/awslabs/sockeye","forks_url":"https://api.github.com/repos/awslabs/sockeye/forks","keys_url":"https://api.github.com/repos/awslabs/sockeye/keys{/key_id}","collaborators_url":"https://api.github.com/repos/awslabs/sockeye/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/awslabs/sockeye/teams","hooks_url":"https://api.github.com/repos/awslabs/sockeye/hooks","issue_events_url":"https://api.github.com/repos/awslabs/sockeye/issues/events{/number}","events_url":"https://api.github.com/repos/awslabs/sockeye/events","assignees_url":"https://api.github.com/repos/awslabs/sockeye/assignees{/user}","branches_url":"https://api.github.com/repos/awslabs/sockeye/branches{/branch}","tags_url":"https://api.github.com/repos/awslabs/sockeye/tags","blobs_url":"https://api.github.com/repos/awslabs/sockeye/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/awslabs/sockeye/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/awslabs/sockeye/git/refs{/sha}","trees_url":"https://api.github.com/repos/awslabs/sockeye/git/trees{/sha}","statuses_url":"https://api.github.com/repos/awslabs/sockeye/statuses/{sha}","languages_url":"https://api.github.com/repos/awslabs/sockeye/languages","stargazers_url":"https://api.github.com/repos/awslabs/sockeye/stargazers","contributors_url":"https://api.github.com/repos/awslabs/sockeye/contributors","subscribers_url":"https://api.github.com/repos/awslabs/sockeye/subscribers","subscription_url":"https://api.github.com/repos/awslabs/sockeye/subscription","commits_url":"https://api.github.com/repos/awslabs/sockeye/commits{/sha}","git_commits_url":"https://api.github.com/repos/awslabs/sockeye/git/commits{/sha}","comments_url":"https://api.github.com/repos/awslabs/sockeye/comments{/number}","issue_comment_url":"https://api.github.com/repos/awslabs/sockeye/issues/comments{/number}","contents_url":"https://api.github.com/repos/awslabs/sockeye/contents/{+path}","compare_url":"https://api.github.com/repos/awslabs/sockeye/compare/{base}...{head}","merges_url":"https://api.github.com/repos/awslabs/sockeye/merges","archive_url":"https://api.github.com/repos/awslabs/sockeye/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/awslabs/sockeye/downloads","issues_url":"https://api.github.com/repos/awslabs/sockeye/issues{/number}","pulls_url":"https://api.github.com/repos/awslabs/sockeye/pulls{/number}","milestones_url":"https://api.github.com/repos/awslabs/sockeye/milestones{/number}","notifications_url":"https://api.github.com/repos/awslabs/sockeye/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/awslabs/sockeye/labels{/name}","releases_url":"https://api.github.com/repos/awslabs/sockeye/releases{/id}","deployments_url":"https://api.github.com/repos/awslabs/sockeye/deployments","created_at":"2017-06-08T07:44:30Z","updated_at":"2017-12-11T15:31:08Z","pushed_at":"2017-12-12T15:25:54Z","git_url":"git://github.com/awslabs/sockeye.git","ssh_url":"git@github.com:awslabs/sockeye.git","clone_url":"https://github.com/awslabs/sockeye.git","svn_url":"https://github.com/awslabs/sockeye","homepage":"","size":1433,"stargazers_count":288,"watchers_count":288,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":79,"mirror_url":null,"archived":false,"open_issues_count":7,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":79,"open_issues":7,"watchers":288,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/241"},"html":{"href":"https://github.com/awslabs/sockeye/pull/241"},"issue":{"href":"https://api.github.com/repos/awslabs/sockeye/issues/241"},"comments":{"href":"https://api.github.com/repos/awslabs/sockeye/issues/241/comments"},"review_comments":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/241/comments"},"review_comment":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/241/commits"},"statuses":{"href":"https://api.github.com/repos/awslabs/sockeye/statuses/3f2742992648f282db6ed9b4947c31c6774ffc1d"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"tdomhan"}}