{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/100083481","pull_request_review_id":20777728,"id":100083481,"diff_hunk":"@@ -1764,4 +1769,125 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {\n     val df2 = spark.read.option(\"PREfersdecimaL\", \"true\").json(records)\n     assert(df2.schema == schema)\n   }\n+\n+  test(\"SPARK-18352: Parse normal multi-line JSON files (compressed)\") {\n+    withTempDir { dir =>\n+      dir.delete()\n+      val path = dir.getCanonicalPath\n+      primitiveFieldAndType\n+        .toDF(\"value\")\n+        .write\n+        .option(\"compression\", \"GzIp\")\n+        .text(path)\n+\n+      new File(path).listFiles() match {\n+        case compressedFiles =>\n+          assert(compressedFiles.exists(_.getName.endsWith(\".gz\")))\n+      }\n+\n+      val jsonDF = spark.read.option(\"wholeFile\", true).json(path)\n+      val jsonDir = new File(dir, \"json\").getCanonicalPath\n+      jsonDF.coalesce(1).write\n+        .format(\"json\")\n+        .option(\"compression\", \"gZiP\")\n+        .save(jsonDir)\n+\n+      new File(jsonDir).listFiles() match {\n+        case compressedFiles =>\n+          assert(compressedFiles.exists(_.getName.endsWith(\".json.gz\")))\n+      }\n+\n+      val jsonCopy = spark.read\n+        .format(\"json\")\n+        .load(jsonDir)\n+\n+      assert(jsonCopy.count === jsonDF.count)\n+      val jsonCopySome = jsonCopy.selectExpr(\"string\", \"long\", \"boolean\")\n+      val jsonDFSome = jsonDF.selectExpr(\"string\", \"long\", \"boolean\")\n+      checkAnswer(jsonCopySome, jsonDFSome)\n+    }\n+  }\n+\n+  test(\"SPARK-18352: Parse normal multi-line JSON files (uncompressed)\") {\n+    withTempDir { dir =>\n+      dir.delete()\n+      val path = dir.getCanonicalPath\n+      primitiveFieldAndType\n+        .toDF(\"value\")\n+        .write\n+        .text(path)\n+\n+      val jsonDF = spark.read.option(\"wholeFile\", true).json(path)\n+      val jsonDir = new File(dir, \"json\").getCanonicalPath\n+      jsonDF.coalesce(1).write\n+        .format(\"json\")\n+        .save(jsonDir)\n+\n+      val compressedFiles = new File(jsonDir).listFiles()\n+      assert(compressedFiles.exists(_.getName.endsWith(\".json\")))\n+\n+      val jsonCopy = spark.read\n+        .format(\"json\")\n+        .load(jsonDir)\n+\n+      assert(jsonCopy.count === jsonDF.count)\n+      val jsonCopySome = jsonCopy.selectExpr(\"string\", \"long\", \"boolean\")\n+      val jsonDFSome = jsonDF.selectExpr(\"string\", \"long\", \"boolean\")\n+      checkAnswer(jsonCopySome, jsonDFSome)\n+    }\n+  }\n+\n+  test(\"SPARK-18352: Expect one JSON document per file\") {\n+    // the json parser terminates as soon as it sees a matching END_OBJECT or END_ARRAY token.\n+    // this might not be the optimal behavior but this test verifies that only the first value\n+    // is parsed and the rest are discarded.\n+\n+    // alternatively the parser could continue parsing following objects, which may further reduce\n+    // allocations by skipping the line reader entirely\n+\n+    withTempDir { dir =>\n+      dir.delete()\n+      val path = dir.getCanonicalPath\n+      primitiveFieldAndType\n+        .flatMap(Iterator.fill(3)(_) ++ Iterator(\"\\n{invalid}\"))","path":"sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala","position":127,"original_position":127,"commit_id":"9dc084d5f0938cc8f5aad1233a99d37bf85957fd","original_commit_id":"9dc084d5f0938cc8f5aad1233a99d37bf85957fd","user":{"login":"cloud-fan","id":3182036,"avatar_url":"https://avatars.githubusercontent.com/u/3182036?v=3","gravatar_id":"","url":"https://api.github.com/users/cloud-fan","html_url":"https://github.com/cloud-fan","followers_url":"https://api.github.com/users/cloud-fan/followers","following_url":"https://api.github.com/users/cloud-fan/following{/other_user}","gists_url":"https://api.github.com/users/cloud-fan/gists{/gist_id}","starred_url":"https://api.github.com/users/cloud-fan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloud-fan/subscriptions","organizations_url":"https://api.github.com/users/cloud-fan/orgs","repos_url":"https://api.github.com/users/cloud-fan/repos","events_url":"https://api.github.com/users/cloud-fan/events{/privacy}","received_events_url":"https://api.github.com/users/cloud-fan/received_events","type":"User","site_admin":false},"body":"can we write json string literal to text file? it's hard to understand what's going on here...","created_at":"2017-02-08T14:58:43Z","updated_at":"2017-02-08T14:58:43Z","html_url":"https://github.com/apache/spark/pull/16386#discussion_r100083481","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/16386","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/100083481"},"html":{"href":"https://github.com/apache/spark/pull/16386#discussion_r100083481"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/16386"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/16386","id":99197588,"html_url":"https://github.com/apache/spark/pull/16386","diff_url":"https://github.com/apache/spark/pull/16386.diff","patch_url":"https://github.com/apache/spark/pull/16386.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/16386","number":16386,"state":"open","locked":false,"title":"[SPARK-18352][SQL] Support parsing multiline json files","user":{"login":"NathanHowell","id":170829,"avatar_url":"https://avatars.githubusercontent.com/u/170829?v=3","gravatar_id":"","url":"https://api.github.com/users/NathanHowell","html_url":"https://github.com/NathanHowell","followers_url":"https://api.github.com/users/NathanHowell/followers","following_url":"https://api.github.com/users/NathanHowell/following{/other_user}","gists_url":"https://api.github.com/users/NathanHowell/gists{/gist_id}","starred_url":"https://api.github.com/users/NathanHowell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NathanHowell/subscriptions","organizations_url":"https://api.github.com/users/NathanHowell/orgs","repos_url":"https://api.github.com/users/NathanHowell/repos","events_url":"https://api.github.com/users/NathanHowell/events{/privacy}","received_events_url":"https://api.github.com/users/NathanHowell/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nIf a new option `wholeFile` is set to `true` the JSON reader will parse each file (instead of a single line) as a value. This is done with Jackson streaming and it should be capable of parsing very large documents, assuming the row will fit in memory.\r\n\r\nBecause the file is not buffered in memory the corrupt record handling is also slightly different when `wholeFile` is enabled: the corrupt column will contain the filename instead of the literal JSON if there is a parsing failure. It would be easy to extend this to add the parser location (line, column and byte offsets) to the output if desired.\r\n\r\nThese changes have allowed types other than `String` to be parsed. Support for `UTF8String` and `Text` have been added (alongside `String` and `InputFormat`) and no longer require a conversion to `String` just for parsing.\r\n\r\nI've also included a few other changes that generate slightly better bytecode and (imo) make it more obvious when and where boxing is occurring in the parser. These are included as separate commits, let me know if they should be flattened into this PR or moved to a new one.\r\n\r\n## How was this patch tested?\r\n\r\nNew and existing unit tests. No performance or load tests have been run.","created_at":"2016-12-23T02:46:32Z","updated_at":"2017-02-08T14:58:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"10404fb676641c603a3ceccd4e971be111ba6b8c","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/16386/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/16386/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/16386/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/9dc084d5f0938cc8f5aad1233a99d37bf85957fd","head":{"label":"NathanHowell:SPARK-18352","ref":"SPARK-18352","sha":"9dc084d5f0938cc8f5aad1233a99d37bf85957fd","user":{"login":"NathanHowell","id":170829,"avatar_url":"https://avatars.githubusercontent.com/u/170829?v=3","gravatar_id":"","url":"https://api.github.com/users/NathanHowell","html_url":"https://github.com/NathanHowell","followers_url":"https://api.github.com/users/NathanHowell/followers","following_url":"https://api.github.com/users/NathanHowell/following{/other_user}","gists_url":"https://api.github.com/users/NathanHowell/gists{/gist_id}","starred_url":"https://api.github.com/users/NathanHowell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NathanHowell/subscriptions","organizations_url":"https://api.github.com/users/NathanHowell/orgs","repos_url":"https://api.github.com/users/NathanHowell/repos","events_url":"https://api.github.com/users/NathanHowell/events{/privacy}","received_events_url":"https://api.github.com/users/NathanHowell/received_events","type":"User","site_admin":false},"repo":{"id":18179192,"name":"spark","full_name":"NathanHowell/spark","owner":{"login":"NathanHowell","id":170829,"avatar_url":"https://avatars.githubusercontent.com/u/170829?v=3","gravatar_id":"","url":"https://api.github.com/users/NathanHowell","html_url":"https://github.com/NathanHowell","followers_url":"https://api.github.com/users/NathanHowell/followers","following_url":"https://api.github.com/users/NathanHowell/following{/other_user}","gists_url":"https://api.github.com/users/NathanHowell/gists{/gist_id}","starred_url":"https://api.github.com/users/NathanHowell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NathanHowell/subscriptions","organizations_url":"https://api.github.com/users/NathanHowell/orgs","repos_url":"https://api.github.com/users/NathanHowell/repos","events_url":"https://api.github.com/users/NathanHowell/events{/privacy}","received_events_url":"https://api.github.com/users/NathanHowell/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/NathanHowell/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/NathanHowell/spark","forks_url":"https://api.github.com/repos/NathanHowell/spark/forks","keys_url":"https://api.github.com/repos/NathanHowell/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/NathanHowell/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/NathanHowell/spark/teams","hooks_url":"https://api.github.com/repos/NathanHowell/spark/hooks","issue_events_url":"https://api.github.com/repos/NathanHowell/spark/issues/events{/number}","events_url":"https://api.github.com/repos/NathanHowell/spark/events","assignees_url":"https://api.github.com/repos/NathanHowell/spark/assignees{/user}","branches_url":"https://api.github.com/repos/NathanHowell/spark/branches{/branch}","tags_url":"https://api.github.com/repos/NathanHowell/spark/tags","blobs_url":"https://api.github.com/repos/NathanHowell/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/NathanHowell/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/NathanHowell/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/NathanHowell/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/NathanHowell/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/NathanHowell/spark/languages","stargazers_url":"https://api.github.com/repos/NathanHowell/spark/stargazers","contributors_url":"https://api.github.com/repos/NathanHowell/spark/contributors","subscribers_url":"https://api.github.com/repos/NathanHowell/spark/subscribers","subscription_url":"https://api.github.com/repos/NathanHowell/spark/subscription","commits_url":"https://api.github.com/repos/NathanHowell/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/NathanHowell/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/NathanHowell/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/NathanHowell/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/NathanHowell/spark/contents/{+path}","compare_url":"https://api.github.com/repos/NathanHowell/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/NathanHowell/spark/merges","archive_url":"https://api.github.com/repos/NathanHowell/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/NathanHowell/spark/downloads","issues_url":"https://api.github.com/repos/NathanHowell/spark/issues{/number}","pulls_url":"https://api.github.com/repos/NathanHowell/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/NathanHowell/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/NathanHowell/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/NathanHowell/spark/labels{/name}","releases_url":"https://api.github.com/repos/NathanHowell/spark/releases{/id}","deployments_url":"https://api.github.com/repos/NathanHowell/spark/deployments","created_at":"2014-03-27T14:43:27Z","updated_at":"2014-03-27T15:13:30Z","pushed_at":"2016-12-29T22:20:35Z","git_url":"git://github.com/NathanHowell/spark.git","ssh_url":"git@github.com:NathanHowell/spark.git","clone_url":"https://github.com/NathanHowell/spark.git","svn_url":"https://github.com/NathanHowell/spark","homepage":null,"size":236917,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"f252cb5d161e064d39cc1ed1d9299307a0636174","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2017-02-08T14:47:15Z","pushed_at":"2017-02-08T14:04:00Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":263684,"stargazers_count":11613,"watchers_count":11613,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":10881,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":474,"forks":10881,"open_issues":474,"watchers":11613,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/16386"},"html":{"href":"https://github.com/apache/spark/pull/16386"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/16386"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/16386/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/16386/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/16386/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/9dc084d5f0938cc8f5aad1233a99d37bf85957fd"}}}}
{"actor":{"display_login":"cloud-fan"}}