{"action":"created","comment":{"url":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments/114013885","pull_request_review_id":35447883,"id":114013885,"diff_hunk":"@@ -221,24 +249,169 @@ public void onTraversalStart() {\n      */\n     @Override\n     public void traverse() {\n-        try {\n-            importer.importBatch();\n-        } catch (IOException e) {\n-            throw new UserException(\"GenomicsDB importBatch raised exception\", e);\n+\n+        if (batchSize == DEFAULT_ZERO_BATCH_SIZE) {\n+            logger.info(\"GenomicsDB import batch started\");\n+            long variantContextBufferSize = vcfBufferSizePerSample * sampleNames.size();\n+\n+            Map<String, FeatureReader<VariantContext>> sampleToReaderMap = getFeatureReaders(sampleNames, sampleNameToFileName, 0, sampleNames.size());\n+\n+            GenomicsDBImportConfiguration.ImportConfiguration importConfiguration =\n+                createImportConfiguration(workspace, arrayName,\n+                    variantContextBufferSize, segmentSize,\n+                    vidMapJSONFile.getAbsolutePath(), callsetMapJSONFile.getAbsolutePath(),\n+                    0, sampleToReaderMap.size() - 1);\n+\n+            try {\n+                importer = new GenomicsDBImporter(sampleToReaderMap, mergedHeader, intervals.get(0), importConfiguration);\n+            } catch (IOException e) {\n+                throw new UserException(\"Error initializing GenomicsDBImporter\", e);\n+            }\n+\n+            try {\n+                importer.importBatch();\n+                assert importer.isDone();\n+            } catch (IOException e) {\n+                throw new UserException(\"GenomicsDB importBatch raised exception\", e);\n+            }\n+\n+            closeReaders(sampleToReaderMap);\n+\n+        } else {\n+            for (int i = 0; i < sampleNames.size(); i += batchSize) {\n+\n+                int upperBoundSampleIndex = min(sampleNames.size(), i+batchSize);\n+                Map<String, FeatureReader<VariantContext>> sampleToReaderMap =\n+                    getFeatureReaders(sampleNames, sampleNameToFileName, i, upperBoundSampleIndex);\n+\n+                logger.info(\"Importing batch \" + i + \" with \" + batchSize + \" samples\");\n+                long variantContextBufferSize = vcfBufferSizePerSample * batchSize;\n+\n+                GenomicsDBImportConfiguration.ImportConfiguration importConfiguration =\n+                        createImportConfiguration(workspace, arrayName,\n+                                variantContextBufferSize, segmentSize,\n+                                \"\", \"\",\n+                                i, upperBoundSampleIndex);\n+\n+                try {\n+                    importer = new GenomicsDBImporter(sampleToReaderMap, mergedHeader, intervals.get(0), importConfiguration);\n+                } catch (IOException e) {\n+                    throw new UserException(\"Error initializing GenomicsDBImporter for batch \" + i +","path":"src/main/java/org/broadinstitute/hellbender/tools/genomicsdb/GenomicsDBImport.java","position":226,"original_position":226,"commit_id":"dff65806d20bdb87544cd4a50002d97650e7a1f7","original_commit_id":"dff65806d20bdb87544cd4a50002d97650e7a1f7","user":{"login":"droazen","id":798637,"avatar_url":"https://avatars3.githubusercontent.com/u/798637?v=3","gravatar_id":"","url":"https://api.github.com/users/droazen","html_url":"https://github.com/droazen","followers_url":"https://api.github.com/users/droazen/followers","following_url":"https://api.github.com/users/droazen/following{/other_user}","gists_url":"https://api.github.com/users/droazen/gists{/gist_id}","starred_url":"https://api.github.com/users/droazen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/droazen/subscriptions","organizations_url":"https://api.github.com/users/droazen/orgs","repos_url":"https://api.github.com/users/droazen/repos","events_url":"https://api.github.com/users/droazen/events{/privacy}","received_events_url":"https://api.github.com/users/droazen/received_events","type":"User","site_admin":false},"body":"Replace `i` with the true batch number in this message, as above.","created_at":"2017-04-28T20:32:04Z","updated_at":"2017-04-28T21:00:48Z","html_url":"https://github.com/broadinstitute/gatk/pull/2634#discussion_r114013885","pull_request_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/2634","_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments/114013885"},"html":{"href":"https://github.com/broadinstitute/gatk/pull/2634#discussion_r114013885"},"pull_request":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/2634"}}},"pull_request":{"url":"https://api.github.com/repos/broadinstitute/gatk/pulls/2634","id":118109674,"html_url":"https://github.com/broadinstitute/gatk/pull/2634","diff_url":"https://github.com/broadinstitute/gatk/pull/2634.diff","patch_url":"https://github.com/broadinstitute/gatk/pull/2634.patch","issue_url":"https://api.github.com/repos/broadinstitute/gatk/issues/2634","number":2634,"state":"open","locked":false,"title":"First commit of add batchsize; build will break","user":{"login":"kdatta","id":4193374,"avatar_url":"https://avatars1.githubusercontent.com/u/4193374?v=3","gravatar_id":"","url":"https://api.github.com/users/kdatta","html_url":"https://github.com/kdatta","followers_url":"https://api.github.com/users/kdatta/followers","following_url":"https://api.github.com/users/kdatta/following{/other_user}","gists_url":"https://api.github.com/users/kdatta/gists{/gist_id}","starred_url":"https://api.github.com/users/kdatta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kdatta/subscriptions","organizations_url":"https://api.github.com/users/kdatta/orgs","repos_url":"https://api.github.com/users/kdatta/repos","events_url":"https://api.github.com/users/kdatta/events{/privacy}","received_events_url":"https://api.github.com/users/kdatta/received_events","type":"User","site_admin":false},"body":"build will break because genomicsdb-0.6.0 hasn't been released yet and dependence on protobuf-java-format needs to be fixed!\r\n\r\n@droazen, remember now that I added the dependence on protobuf-java-format to use protobuf.JsonFormat.printToString() method which converts a protobuf structure to JSON string. I want to use import configuration protocol buffers in this code which means this dependence will be back to bite us! Need to fix this cause I don't want to break the Spark build again","created_at":"2017-04-28T09:07:24Z","updated_at":"2017-04-28T21:00:48Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e6b54b08e30df8b22fbb956198b2a1cb20c783ff","assignee":{"login":"droazen","id":798637,"avatar_url":"https://avatars3.githubusercontent.com/u/798637?v=3","gravatar_id":"","url":"https://api.github.com/users/droazen","html_url":"https://github.com/droazen","followers_url":"https://api.github.com/users/droazen/followers","following_url":"https://api.github.com/users/droazen/following{/other_user}","gists_url":"https://api.github.com/users/droazen/gists{/gist_id}","starred_url":"https://api.github.com/users/droazen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/droazen/subscriptions","organizations_url":"https://api.github.com/users/droazen/orgs","repos_url":"https://api.github.com/users/droazen/repos","events_url":"https://api.github.com/users/droazen/events{/privacy}","received_events_url":"https://api.github.com/users/droazen/received_events","type":"User","site_admin":false},"assignees":[{"login":"droazen","id":798637,"avatar_url":"https://avatars3.githubusercontent.com/u/798637?v=3","gravatar_id":"","url":"https://api.github.com/users/droazen","html_url":"https://github.com/droazen","followers_url":"https://api.github.com/users/droazen/followers","following_url":"https://api.github.com/users/droazen/following{/other_user}","gists_url":"https://api.github.com/users/droazen/gists{/gist_id}","starred_url":"https://api.github.com/users/droazen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/droazen/subscriptions","organizations_url":"https://api.github.com/users/droazen/orgs","repos_url":"https://api.github.com/users/droazen/repos","events_url":"https://api.github.com/users/droazen/events{/privacy}","received_events_url":"https://api.github.com/users/droazen/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/2634/commits","review_comments_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/2634/comments","review_comment_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments{/number}","comments_url":"https://api.github.com/repos/broadinstitute/gatk/issues/2634/comments","statuses_url":"https://api.github.com/repos/broadinstitute/gatk/statuses/dff65806d20bdb87544cd4a50002d97650e7a1f7","head":{"label":"kdatta:issue-2613","ref":"issue-2613","sha":"dff65806d20bdb87544cd4a50002d97650e7a1f7","user":{"login":"kdatta","id":4193374,"avatar_url":"https://avatars1.githubusercontent.com/u/4193374?v=3","gravatar_id":"","url":"https://api.github.com/users/kdatta","html_url":"https://github.com/kdatta","followers_url":"https://api.github.com/users/kdatta/followers","following_url":"https://api.github.com/users/kdatta/following{/other_user}","gists_url":"https://api.github.com/users/kdatta/gists{/gist_id}","starred_url":"https://api.github.com/users/kdatta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kdatta/subscriptions","organizations_url":"https://api.github.com/users/kdatta/orgs","repos_url":"https://api.github.com/users/kdatta/repos","events_url":"https://api.github.com/users/kdatta/events{/privacy}","received_events_url":"https://api.github.com/users/kdatta/received_events","type":"User","site_admin":false},"repo":{"id":68648948,"name":"gatk","full_name":"kdatta/gatk","owner":{"login":"kdatta","id":4193374,"avatar_url":"https://avatars1.githubusercontent.com/u/4193374?v=3","gravatar_id":"","url":"https://api.github.com/users/kdatta","html_url":"https://github.com/kdatta","followers_url":"https://api.github.com/users/kdatta/followers","following_url":"https://api.github.com/users/kdatta/following{/other_user}","gists_url":"https://api.github.com/users/kdatta/gists{/gist_id}","starred_url":"https://api.github.com/users/kdatta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kdatta/subscriptions","organizations_url":"https://api.github.com/users/kdatta/orgs","repos_url":"https://api.github.com/users/kdatta/repos","events_url":"https://api.github.com/users/kdatta/events{/privacy}","received_events_url":"https://api.github.com/users/kdatta/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kdatta/gatk","description":"GATK4 development","fork":true,"url":"https://api.github.com/repos/kdatta/gatk","forks_url":"https://api.github.com/repos/kdatta/gatk/forks","keys_url":"https://api.github.com/repos/kdatta/gatk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kdatta/gatk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kdatta/gatk/teams","hooks_url":"https://api.github.com/repos/kdatta/gatk/hooks","issue_events_url":"https://api.github.com/repos/kdatta/gatk/issues/events{/number}","events_url":"https://api.github.com/repos/kdatta/gatk/events","assignees_url":"https://api.github.com/repos/kdatta/gatk/assignees{/user}","branches_url":"https://api.github.com/repos/kdatta/gatk/branches{/branch}","tags_url":"https://api.github.com/repos/kdatta/gatk/tags","blobs_url":"https://api.github.com/repos/kdatta/gatk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kdatta/gatk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kdatta/gatk/git/refs{/sha}","trees_url":"https://api.github.com/repos/kdatta/gatk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kdatta/gatk/statuses/{sha}","languages_url":"https://api.github.com/repos/kdatta/gatk/languages","stargazers_url":"https://api.github.com/repos/kdatta/gatk/stargazers","contributors_url":"https://api.github.com/repos/kdatta/gatk/contributors","subscribers_url":"https://api.github.com/repos/kdatta/gatk/subscribers","subscription_url":"https://api.github.com/repos/kdatta/gatk/subscription","commits_url":"https://api.github.com/repos/kdatta/gatk/commits{/sha}","git_commits_url":"https://api.github.com/repos/kdatta/gatk/git/commits{/sha}","comments_url":"https://api.github.com/repos/kdatta/gatk/comments{/number}","issue_comment_url":"https://api.github.com/repos/kdatta/gatk/issues/comments{/number}","contents_url":"https://api.github.com/repos/kdatta/gatk/contents/{+path}","compare_url":"https://api.github.com/repos/kdatta/gatk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kdatta/gatk/merges","archive_url":"https://api.github.com/repos/kdatta/gatk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kdatta/gatk/downloads","issues_url":"https://api.github.com/repos/kdatta/gatk/issues{/number}","pulls_url":"https://api.github.com/repos/kdatta/gatk/pulls{/number}","milestones_url":"https://api.github.com/repos/kdatta/gatk/milestones{/number}","notifications_url":"https://api.github.com/repos/kdatta/gatk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kdatta/gatk/labels{/name}","releases_url":"https://api.github.com/repos/kdatta/gatk/releases{/id}","deployments_url":"https://api.github.com/repos/kdatta/gatk/deployments","created_at":"2016-09-19T21:38:19Z","updated_at":"2016-09-19T21:38:29Z","pushed_at":"2017-04-28T09:44:11Z","git_url":"git://github.com/kdatta/gatk.git","ssh_url":"git@github.com:kdatta/gatk.git","clone_url":"https://github.com/kdatta/gatk.git","svn_url":"https://github.com/kdatta/gatk","homepage":"","size":77852,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"broadinstitute:master","ref":"master","sha":"1dde31b4045c8b0b4edfedc2ac1c7f58da30db30","user":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars3.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"repo":{"id":27452807,"name":"gatk","full_name":"broadinstitute/gatk","owner":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars3.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/broadinstitute/gatk","description":"GATK4 development","fork":false,"url":"https://api.github.com/repos/broadinstitute/gatk","forks_url":"https://api.github.com/repos/broadinstitute/gatk/forks","keys_url":"https://api.github.com/repos/broadinstitute/gatk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/broadinstitute/gatk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/broadinstitute/gatk/teams","hooks_url":"https://api.github.com/repos/broadinstitute/gatk/hooks","issue_events_url":"https://api.github.com/repos/broadinstitute/gatk/issues/events{/number}","events_url":"https://api.github.com/repos/broadinstitute/gatk/events","assignees_url":"https://api.github.com/repos/broadinstitute/gatk/assignees{/user}","branches_url":"https://api.github.com/repos/broadinstitute/gatk/branches{/branch}","tags_url":"https://api.github.com/repos/broadinstitute/gatk/tags","blobs_url":"https://api.github.com/repos/broadinstitute/gatk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/broadinstitute/gatk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/broadinstitute/gatk/git/refs{/sha}","trees_url":"https://api.github.com/repos/broadinstitute/gatk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/broadinstitute/gatk/statuses/{sha}","languages_url":"https://api.github.com/repos/broadinstitute/gatk/languages","stargazers_url":"https://api.github.com/repos/broadinstitute/gatk/stargazers","contributors_url":"https://api.github.com/repos/broadinstitute/gatk/contributors","subscribers_url":"https://api.github.com/repos/broadinstitute/gatk/subscribers","subscription_url":"https://api.github.com/repos/broadinstitute/gatk/subscription","commits_url":"https://api.github.com/repos/broadinstitute/gatk/commits{/sha}","git_commits_url":"https://api.github.com/repos/broadinstitute/gatk/git/commits{/sha}","comments_url":"https://api.github.com/repos/broadinstitute/gatk/comments{/number}","issue_comment_url":"https://api.github.com/repos/broadinstitute/gatk/issues/comments{/number}","contents_url":"https://api.github.com/repos/broadinstitute/gatk/contents/{+path}","compare_url":"https://api.github.com/repos/broadinstitute/gatk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/broadinstitute/gatk/merges","archive_url":"https://api.github.com/repos/broadinstitute/gatk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/broadinstitute/gatk/downloads","issues_url":"https://api.github.com/repos/broadinstitute/gatk/issues{/number}","pulls_url":"https://api.github.com/repos/broadinstitute/gatk/pulls{/number}","milestones_url":"https://api.github.com/repos/broadinstitute/gatk/milestones{/number}","notifications_url":"https://api.github.com/repos/broadinstitute/gatk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/broadinstitute/gatk/labels{/name}","releases_url":"https://api.github.com/repos/broadinstitute/gatk/releases{/id}","deployments_url":"https://api.github.com/repos/broadinstitute/gatk/deployments","created_at":"2014-12-02T20:50:36Z","updated_at":"2017-04-27T08:44:44Z","pushed_at":"2017-04-28T19:41:05Z","git_url":"git://github.com/broadinstitute/gatk.git","ssh_url":"git@github.com:broadinstitute/gatk.git","clone_url":"https://github.com/broadinstitute/gatk.git","svn_url":"https://github.com/broadinstitute/gatk","homepage":"","size":78838,"stargazers_count":114,"watchers_count":114,"language":"Java","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":50,"mirror_url":null,"open_issues_count":435,"forks":50,"open_issues":435,"watchers":114,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/2634"},"html":{"href":"https://github.com/broadinstitute/gatk/pull/2634"},"issue":{"href":"https://api.github.com/repos/broadinstitute/gatk/issues/2634"},"comments":{"href":"https://api.github.com/repos/broadinstitute/gatk/issues/2634/comments"},"review_comments":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/2634/comments"},"review_comment":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/2634/commits"},"statuses":{"href":"https://api.github.com/repos/broadinstitute/gatk/statuses/dff65806d20bdb87544cd4a50002d97650e7a1f7"}}}}
{"actor":{"display_login":"droazen"}}