{"action":"created","comment":{"url":"https://api.github.com/repos/palantir/spark/pulls/comments/124155454","pull_request_review_id":46420425,"id":124155454,"diff_hunk":"@@ -122,7 +122,14 @@ object FileFormatWriter extends Logging {\n       // guarantee the data distribution is same between shuffle and bucketed data source, which\n       // enables us to only shuffle one side when join a bucketed table and a normal one.\n       HashPartitioning(bucketColumns, spec.numBuckets).partitionIdExpression\n-    }\n+    }.orElse(\n+      // If there are no columns in the bucket spec, check if the data is already HashPartitioned\n+      // and use that HashPartitioning's partitionIdExpression\n+      plan.outputPartitioning match {","path":"sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala","position":8,"original_position":8,"commit_id":"a685b502d8e7a2c1828ec2acb1ad96c0c654d10b","original_commit_id":"a685b502d8e7a2c1828ec2acb1ad96c0c654d10b","user":{"login":"robert3005","id":512084,"avatar_url":"https://avatars3.githubusercontent.com/u/512084?v=3","gravatar_id":"","url":"https://api.github.com/users/robert3005","html_url":"https://github.com/robert3005","followers_url":"https://api.github.com/users/robert3005/followers","following_url":"https://api.github.com/users/robert3005/following{/other_user}","gists_url":"https://api.github.com/users/robert3005/gists{/gist_id}","starred_url":"https://api.github.com/users/robert3005/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robert3005/subscriptions","organizations_url":"https://api.github.com/users/robert3005/orgs","repos_url":"https://api.github.com/users/robert3005/repos","events_url":"https://api.github.com/users/robert3005/events{/privacy}","received_events_url":"https://api.github.com/users/robert3005/received_events","type":"User","site_admin":false},"body":"Sorry, had to http://blog.originate.com/blog/2014/06/15/idiomatic-scala-your-options-do-not-match/.\r\n\r\nSpark prefers map.getOrElse.","created_at":"2017-06-27T00:44:33Z","updated_at":"2017-06-27T00:44:33Z","html_url":"https://github.com/palantir/spark/pull/218#discussion_r124155454","pull_request_url":"https://api.github.com/repos/palantir/spark/pulls/218","_links":{"self":{"href":"https://api.github.com/repos/palantir/spark/pulls/comments/124155454"},"html":{"href":"https://github.com/palantir/spark/pull/218#discussion_r124155454"},"pull_request":{"href":"https://api.github.com/repos/palantir/spark/pulls/218"}}},"pull_request":{"url":"https://api.github.com/repos/palantir/spark/pulls/218","id":127587909,"html_url":"https://github.com/palantir/spark/pull/218","diff_url":"https://github.com/palantir/spark/pull/218.diff","patch_url":"https://github.com/palantir/spark/pull/218.patch","issue_url":"https://api.github.com/repos/palantir/spark/issues/218","number":218,"state":"open","locked":false,"title":"SPARK-21220 Use outputPartitioning as bucketing if possible","user":{"login":"ash211","id":357170,"avatar_url":"https://avatars1.githubusercontent.com/u/357170?v=3","gravatar_id":"","url":"https://api.github.com/users/ash211","html_url":"https://github.com/ash211","followers_url":"https://api.github.com/users/ash211/followers","following_url":"https://api.github.com/users/ash211/following{/other_user}","gists_url":"https://api.github.com/users/ash211/gists{/gist_id}","starred_url":"https://api.github.com/users/ash211/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ash211/subscriptions","organizations_url":"https://api.github.com/users/ash211/orgs","repos_url":"https://api.github.com/users/ash211/repos","events_url":"https://api.github.com/users/ash211/events{/privacy}","received_events_url":"https://api.github.com/users/ash211/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nIf possible, set the bucketing columns to be the outputPartitioning if it's a HashPartitioning\r\n\r\n## How was this patch tested?\r\n\r\nTODO","created_at":"2017-06-27T00:02:51Z","updated_at":"2017-06-27T00:44:33Z","closed_at":null,"merged_at":null,"merge_commit_sha":"af5cee202bddcc72a415e4912c3b7c0bdbef91e6","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/palantir/spark/pulls/218/commits","review_comments_url":"https://api.github.com/repos/palantir/spark/pulls/218/comments","review_comment_url":"https://api.github.com/repos/palantir/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/palantir/spark/issues/218/comments","statuses_url":"https://api.github.com/repos/palantir/spark/statuses/a685b502d8e7a2c1828ec2acb1ad96c0c654d10b","head":{"label":"palantir:SPARK-21220-outputpartitioning-as-bucketing","ref":"SPARK-21220-outputpartitioning-as-bucketing","sha":"a685b502d8e7a2c1828ec2acb1ad96c0c654d10b","user":{"login":"palantir","id":303157,"avatar_url":"https://avatars3.githubusercontent.com/u/303157?v=3","gravatar_id":"","url":"https://api.github.com/users/palantir","html_url":"https://github.com/palantir","followers_url":"https://api.github.com/users/palantir/followers","following_url":"https://api.github.com/users/palantir/following{/other_user}","gists_url":"https://api.github.com/users/palantir/gists{/gist_id}","starred_url":"https://api.github.com/users/palantir/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/palantir/subscriptions","organizations_url":"https://api.github.com/users/palantir/orgs","repos_url":"https://api.github.com/users/palantir/repos","events_url":"https://api.github.com/users/palantir/events{/privacy}","received_events_url":"https://api.github.com/users/palantir/received_events","type":"Organization","site_admin":false},"repo":{"id":30919203,"name":"spark","full_name":"palantir/spark","owner":{"login":"palantir","id":303157,"avatar_url":"https://avatars3.githubusercontent.com/u/303157?v=3","gravatar_id":"","url":"https://api.github.com/users/palantir","html_url":"https://github.com/palantir","followers_url":"https://api.github.com/users/palantir/followers","following_url":"https://api.github.com/users/palantir/following{/other_user}","gists_url":"https://api.github.com/users/palantir/gists{/gist_id}","starred_url":"https://api.github.com/users/palantir/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/palantir/subscriptions","organizations_url":"https://api.github.com/users/palantir/orgs","repos_url":"https://api.github.com/users/palantir/repos","events_url":"https://api.github.com/users/palantir/events{/privacy}","received_events_url":"https://api.github.com/users/palantir/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/palantir/spark","description":"Palantir Distribution of Apache Spark","fork":true,"url":"https://api.github.com/repos/palantir/spark","forks_url":"https://api.github.com/repos/palantir/spark/forks","keys_url":"https://api.github.com/repos/palantir/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/palantir/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/palantir/spark/teams","hooks_url":"https://api.github.com/repos/palantir/spark/hooks","issue_events_url":"https://api.github.com/repos/palantir/spark/issues/events{/number}","events_url":"https://api.github.com/repos/palantir/spark/events","assignees_url":"https://api.github.com/repos/palantir/spark/assignees{/user}","branches_url":"https://api.github.com/repos/palantir/spark/branches{/branch}","tags_url":"https://api.github.com/repos/palantir/spark/tags","blobs_url":"https://api.github.com/repos/palantir/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/palantir/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/palantir/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/palantir/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/palantir/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/palantir/spark/languages","stargazers_url":"https://api.github.com/repos/palantir/spark/stargazers","contributors_url":"https://api.github.com/repos/palantir/spark/contributors","subscribers_url":"https://api.github.com/repos/palantir/spark/subscribers","subscription_url":"https://api.github.com/repos/palantir/spark/subscription","commits_url":"https://api.github.com/repos/palantir/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/palantir/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/palantir/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/palantir/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/palantir/spark/contents/{+path}","compare_url":"https://api.github.com/repos/palantir/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/palantir/spark/merges","archive_url":"https://api.github.com/repos/palantir/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/palantir/spark/downloads","issues_url":"https://api.github.com/repos/palantir/spark/issues{/number}","pulls_url":"https://api.github.com/repos/palantir/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/palantir/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/palantir/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/palantir/spark/labels{/name}","releases_url":"https://api.github.com/repos/palantir/spark/releases{/id}","deployments_url":"https://api.github.com/repos/palantir/spark/deployments","created_at":"2015-02-17T13:57:35Z","updated_at":"2017-05-27T17:05:44Z","pushed_at":"2017-06-27T00:02:51Z","git_url":"git://github.com/palantir/spark.git","ssh_url":"git@github.com:palantir/spark.git","clone_url":"https://github.com/palantir/spark.git","svn_url":"https://github.com/palantir/spark","homepage":"","size":273263,"stargazers_count":16,"watchers_count":16,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":13,"mirror_url":null,"open_issues_count":20,"forks":13,"open_issues":20,"watchers":16,"default_branch":"master"}},"base":{"label":"palantir:master","ref":"master","sha":"4144776488d3fd4475110610ca1a654d695cc883","user":{"login":"palantir","id":303157,"avatar_url":"https://avatars3.githubusercontent.com/u/303157?v=3","gravatar_id":"","url":"https://api.github.com/users/palantir","html_url":"https://github.com/palantir","followers_url":"https://api.github.com/users/palantir/followers","following_url":"https://api.github.com/users/palantir/following{/other_user}","gists_url":"https://api.github.com/users/palantir/gists{/gist_id}","starred_url":"https://api.github.com/users/palantir/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/palantir/subscriptions","organizations_url":"https://api.github.com/users/palantir/orgs","repos_url":"https://api.github.com/users/palantir/repos","events_url":"https://api.github.com/users/palantir/events{/privacy}","received_events_url":"https://api.github.com/users/palantir/received_events","type":"Organization","site_admin":false},"repo":{"id":30919203,"name":"spark","full_name":"palantir/spark","owner":{"login":"palantir","id":303157,"avatar_url":"https://avatars3.githubusercontent.com/u/303157?v=3","gravatar_id":"","url":"https://api.github.com/users/palantir","html_url":"https://github.com/palantir","followers_url":"https://api.github.com/users/palantir/followers","following_url":"https://api.github.com/users/palantir/following{/other_user}","gists_url":"https://api.github.com/users/palantir/gists{/gist_id}","starred_url":"https://api.github.com/users/palantir/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/palantir/subscriptions","organizations_url":"https://api.github.com/users/palantir/orgs","repos_url":"https://api.github.com/users/palantir/repos","events_url":"https://api.github.com/users/palantir/events{/privacy}","received_events_url":"https://api.github.com/users/palantir/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/palantir/spark","description":"Palantir Distribution of Apache Spark","fork":true,"url":"https://api.github.com/repos/palantir/spark","forks_url":"https://api.github.com/repos/palantir/spark/forks","keys_url":"https://api.github.com/repos/palantir/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/palantir/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/palantir/spark/teams","hooks_url":"https://api.github.com/repos/palantir/spark/hooks","issue_events_url":"https://api.github.com/repos/palantir/spark/issues/events{/number}","events_url":"https://api.github.com/repos/palantir/spark/events","assignees_url":"https://api.github.com/repos/palantir/spark/assignees{/user}","branches_url":"https://api.github.com/repos/palantir/spark/branches{/branch}","tags_url":"https://api.github.com/repos/palantir/spark/tags","blobs_url":"https://api.github.com/repos/palantir/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/palantir/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/palantir/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/palantir/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/palantir/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/palantir/spark/languages","stargazers_url":"https://api.github.com/repos/palantir/spark/stargazers","contributors_url":"https://api.github.com/repos/palantir/spark/contributors","subscribers_url":"https://api.github.com/repos/palantir/spark/subscribers","subscription_url":"https://api.github.com/repos/palantir/spark/subscription","commits_url":"https://api.github.com/repos/palantir/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/palantir/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/palantir/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/palantir/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/palantir/spark/contents/{+path}","compare_url":"https://api.github.com/repos/palantir/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/palantir/spark/merges","archive_url":"https://api.github.com/repos/palantir/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/palantir/spark/downloads","issues_url":"https://api.github.com/repos/palantir/spark/issues{/number}","pulls_url":"https://api.github.com/repos/palantir/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/palantir/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/palantir/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/palantir/spark/labels{/name}","releases_url":"https://api.github.com/repos/palantir/spark/releases{/id}","deployments_url":"https://api.github.com/repos/palantir/spark/deployments","created_at":"2015-02-17T13:57:35Z","updated_at":"2017-05-27T17:05:44Z","pushed_at":"2017-06-27T00:02:51Z","git_url":"git://github.com/palantir/spark.git","ssh_url":"git@github.com:palantir/spark.git","clone_url":"https://github.com/palantir/spark.git","svn_url":"https://github.com/palantir/spark","homepage":"","size":273263,"stargazers_count":16,"watchers_count":16,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":13,"mirror_url":null,"open_issues_count":20,"forks":13,"open_issues":20,"watchers":16,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/palantir/spark/pulls/218"},"html":{"href":"https://github.com/palantir/spark/pull/218"},"issue":{"href":"https://api.github.com/repos/palantir/spark/issues/218"},"comments":{"href":"https://api.github.com/repos/palantir/spark/issues/218/comments"},"review_comments":{"href":"https://api.github.com/repos/palantir/spark/pulls/218/comments"},"review_comment":{"href":"https://api.github.com/repos/palantir/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/palantir/spark/pulls/218/commits"},"statuses":{"href":"https://api.github.com/repos/palantir/spark/statuses/a685b502d8e7a2c1828ec2acb1ad96c0c654d10b"}}}}
{"actor":{"display_login":"robert3005"}}