{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/103054126","pull_request_review_id":23824005,"id":103054126,"diff_hunk":"@@ -0,0 +1,413 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.kafka010\n+\n+import java.util.concurrent.atomic.AtomicInteger\n+\n+import org.scalatest.time.SpanSugar._\n+\n+import org.apache.spark.SparkException\n+import org.apache.spark.sql.{AnalysisException, SaveMode}\n+import org.apache.spark.sql.catalyst.InternalRow\n+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, SpecificInternalRow, UnsafeProjection}\n+import org.apache.spark.sql.execution.streaming.MemoryStream\n+import org.apache.spark.sql.streaming._\n+import org.apache.spark.sql.test.SharedSQLContext\n+import org.apache.spark.sql.types.{BinaryType, DataType}\n+\n+class KafkaSinkSuite extends StreamTest with SharedSQLContext {\n+  import testImplicits._\n+\n+  case class AddMoreData(\n+      ms: MemoryStream[String],\n+      q: StreamingQuery,\n+      values: String*) extends ExternalAction {\n+    override def runAction(): Unit = {\n+      ms.addData(values)\n+      q.processAllAvailable()\n+      Thread.sleep(5000) // wait for data to appear in Kafka\n+    }\n+  }\n+\n+  protected var testUtils: KafkaTestUtils = _\n+\n+  override val streamingTimeout = 30.seconds\n+\n+  override def beforeAll(): Unit = {\n+    super.beforeAll()\n+    testUtils = new KafkaTestUtils(\n+      withBrokerProps = Map(\"auto.create.topics.enable\" -> \"false\"))\n+    testUtils.setup()\n+  }\n+\n+  override def afterAll(): Unit = {\n+    if (testUtils != null) {\n+      testUtils.teardown()\n+      testUtils = null\n+      super.afterAll()\n+    }\n+  }\n+\n+  private val topicId = new AtomicInteger(0)\n+\n+  private def newTopic(): String = s\"topic-${topicId.getAndIncrement()}\"\n+\n+  test(\"write to stream with topic field\") {\n+    withTempDir { checkpointDir =>\n+      val input = MemoryStream[String]\n+      val topic = newTopic()\n+      testUtils.createTopic(topic)\n+\n+      val writer = input.toDF()\n+        .selectExpr(s\"'$topic' as topic\", \"value\")\n+        .writeStream\n+        .format(\"kafka\")\n+        .option(\"checkpointLocation\", checkpointDir.getCanonicalPath)\n+        .outputMode(OutputMode.Append)\n+        .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+        .queryName(\"kafkaStream\")\n+        .start()\n+\n+      // Create Kafka source that reads from earliest to latest offset\n+      val reader = spark.readStream\n+        .format(\"kafka\")\n+        .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+        .option(\"kafka.metadata.max.age.ms\", \"1\")\n+        .option(\"startingOffsets\", \"earliest\")\n+        .option(\"subscribe\", topic)\n+        .option(\"failOnDataLoss\", \"true\")\n+        .load()\n+      val kafka = reader\n+        .selectExpr(\"CAST(key AS STRING)\", \"CAST(value AS STRING)\")\n+        .as[(String, String)]\n+      val mapped: org.apache.spark.sql.Dataset[_] = kafka.map(kv => kv._2.trim.toInt)\n+\n+      testStream(mapped, outputMode = OutputMode.Append)(\n+        StartStream(ProcessingTime(0)),\n+        AddMoreData(input, writer, \"1\", \"2\", \"3\", \"4\", \"5\"),\n+\n+        CheckAnswer(1, 2, 3, 4, 5),\n+        AddMoreData(input, writer, \"6\", \"7\", \"8\", \"9\", \"10\"),\n+        CheckAnswer(1, 2, 3, 4, 5, 6, 7, 8, 9, 10),\n+        StopStream\n+      )\n+      writer.stop()\n+    }\n+  }\n+\n+  test(\"write structured streaming aggregation w/o topic field, with topic option\") {\n+    withTempDir { checkpointDir =>\n+      val input = MemoryStream[String]\n+      val topic = newTopic()\n+      testUtils.createTopic(topic)\n+\n+      val writer = input.toDF()\n+        .groupBy(\"value\")\n+        .count()\n+        .selectExpr(\"CAST(value as STRING) key\", \"CAST(count as STRING) value\")\n+        .writeStream\n+        .format(\"kafka\")\n+        .option(\"checkpointLocation\", checkpointDir.getCanonicalPath)\n+        .outputMode(OutputMode.Update)\n+        .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+        .option(\"topic\", topic)\n+        .queryName(\"kafkaAggStream\")\n+        .start()\n+\n+      // Create Kafka source that reads from earliest to latest offset\n+      val reader = spark.readStream\n+        .format(\"kafka\")\n+        .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+        .option(\"kafka.metadata.max.age.ms\", \"1\")\n+        .option(\"startingOffsets\", \"earliest\")\n+        .option(\"subscribe\", topic)\n+        .option(\"failOnDataLoss\", \"true\")\n+        .load()\n+      val kafka = reader\n+        .selectExpr(\"CAST(key AS STRING)\", \"CAST(value AS STRING)\")\n+        .selectExpr(\"CAST(key AS INT)\", \"CAST(value AS INT)\")\n+        .as[(Int, Int)]\n+\n+      testStream(kafka, outputMode = OutputMode.Update)(\n+        StartStream(ProcessingTime(0)),\n+        AddMoreData(input, writer, \"1\", \"2\", \"2\", \"3\", \"3\", \"3\"),\n+\n+        CheckAnswer((1, 1), (2, 2), (3, 3)),\n+        AddMoreData(input, writer, \"1\", \"2\", \"3\"),\n+        CheckAnswer((1, 1), (2, 2), (3, 3), (1, 2), (2, 3), (3, 4)),\n+        StopStream\n+      )\n+      writer.stop()\n+    }\n+  }\n+\n+  test(\"write structured streaming aggregation with topic field and topic option\") {\n+    /* The purpose of this test is to ensure that the topic option\n+     * overrides the topic field. We begin by writing some data that\n+     * includes a topic field and value (e.g., 'foo') along with a topic\n+     * option. Then when we read from the topic specified in the option\n+     * we should see the data i.e., the data was written to the topic\n+     * option, and not to the topic in the data e.g., foo\n+     */\n+    withTempDir { checkpointDir =>\n+      val input = MemoryStream[String]\n+      val topic = newTopic()\n+      testUtils.createTopic(topic)\n+\n+      val writer = input.toDF()\n+        .groupBy(\"value\")\n+        .count()\n+        .selectExpr(\"'foo' as topic\",\n+          \"CAST(value as STRING) key\", \"CAST(count as STRING) value\")\n+        .writeStream\n+        .format(\"kafka\")\n+        .option(\"checkpointLocation\", checkpointDir.getCanonicalPath)\n+        .outputMode(OutputMode.Update)\n+        .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+        .option(\"topic\", topic)\n+        .queryName(\"kafkaAggStream\")\n+        .start()\n+\n+      // Create Kafka source that reads from earliest to latest offset\n+      val reader = spark.readStream\n+        .format(\"kafka\")\n+        .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+        .option(\"kafka.metadata.max.age.ms\", \"1\")\n+        .option(\"startingOffsets\", \"earliest\")\n+        .option(\"subscribe\", topic)\n+        .option(\"failOnDataLoss\", \"true\")\n+        .load()\n+      val kafka = reader\n+        .selectExpr(\"CAST(key AS STRING)\", \"CAST(value AS STRING)\")\n+        .selectExpr(\"CAST(key AS INT)\", \"CAST(value AS INT)\")\n+        .as[(Int, Int)]\n+\n+      testStream(kafka, outputMode = OutputMode.Update)(\n+        StartStream(ProcessingTime(0)),\n+        AddMoreData(input, writer, \"1\", \"2\", \"2\", \"3\", \"3\", \"3\"),\n+\n+        CheckAnswer((1, 1), (2, 2), (3, 3)),\n+        AddMoreData(input, writer, \"1\", \"2\", \"3\"),\n+        CheckAnswer((1, 1), (2, 2), (3, 3), (1, 2), (2, 3), (3, 4)),\n+        StopStream\n+      )\n+      writer.stop()\n+    }\n+  }\n+\n+\n+  test(\"write data with bad schema\") {\n+    withTempDir { checkpointDir =>\n+      val input = MemoryStream[String]\n+      val topic = newTopic()\n+      testUtils.createTopic(topic)\n+\n+      /* No topic field or topic option */\n+      var writer: StreamingQuery = null\n+      var ex = intercept[StreamingQueryException] {\n+        writer = input.toDF()\n+          .selectExpr(\"value as key\", \"value\")\n+          .writeStream\n+          .format(\"kafka\")\n+          .option(\"checkpointLocation\", checkpointDir.getCanonicalPath)\n+          .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+          .queryName(\"kafkaNoTopicFieldStream\")\n+          .start()\n+        input.addData(\"1\", \"2\", \"3\", \"4\", \"5\")\n+        writer.processAllAvailable()\n+      }\n+      writer.stop()\n+      assert(ex.getMessage\n+        .toLowerCase\n+        .contains(\"topic option required when no 'topic' attribute is present\"))\n+\n+      /* No value field */\n+      ex = intercept[StreamingQueryException] {\n+        writer = input.toDF()\n+          .selectExpr(s\"'$topic' as topic\", \"value as key\")\n+          .writeStream\n+          .format(\"kafka\")\n+          .option(\"checkpointLocation\", checkpointDir.getCanonicalPath)\n+          .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+          .queryName(\"kafkaNoValueFieldStream\")\n+          .start()\n+        input.addData(\"1\", \"2\", \"3\", \"4\", \"5\")\n+        writer.processAllAvailable()\n+      }\n+      writer.stop()\n+      assert(ex.getMessage.toLowerCase.contains(\"required attribute 'value' not found\"))\n+    }\n+  }\n+\n+  test(\"write data with valid schema but wrong types\") {\n+    withTempDir { checkpointDir =>\n+      val input = MemoryStream[String]\n+      val topic = newTopic()\n+      testUtils.createTopic(topic)\n+\n+      /* value field wrong type */\n+      var writer: StreamingQuery = null\n+      var ex = intercept[StreamingQueryException] {\n+        writer = input.toDF()\n+          .selectExpr(s\"'$topic' as topic\", \"CAST(value as INT) as value\")\n+          .writeStream\n+          .format(\"kafka\")\n+          .option(\"checkpointLocation\", checkpointDir.getCanonicalPath)\n+          .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+          .queryName(\"kafkaIntValueFieldStream\")\n+          .start()\n+        input.addData(\"1\", \"2\", \"3\", \"4\", \"5\")\n+        writer.processAllAvailable()\n+      }\n+      writer.stop()\n+      assert(ex.getMessage.toLowerCase.contains(\n+        \"value attribute type must be a string or binarytype\"))\n+\n+      /* key field wrong type */\n+      ex = intercept[StreamingQueryException] {\n+        writer = input.toDF()\n+          .selectExpr(s\"'$topic' as topic\", \"CAST(value as INT) as key\", \"value\")\n+          .writeStream\n+          .format(\"kafka\")\n+          .option(\"checkpointLocation\", checkpointDir.getCanonicalPath)\n+          .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+          .queryName(\"kafkaIntValueFieldStream\")\n+          .start()\n+        input.addData(\"1\", \"2\", \"3\", \"4\", \"5\")\n+        writer.processAllAvailable()\n+      }\n+      writer.stop()\n+      assert(ex.getMessage.toLowerCase.contains(\n+        \"key attribute type must be a string or binarytype\"))\n+    }\n+  }\n+\n+  test(\"write to non-existing topic\") {\n+    withTempDir { checkpointDir =>\n+      val input = MemoryStream[String]\n+      val topic = newTopic()\n+\n+      var writer: StreamingQuery = null\n+      val ex = intercept[StreamingQueryException] {\n+        writer = input.toDF()\n+          .writeStream\n+          .format(\"kafka\")\n+          .option(\"checkpointLocation\", checkpointDir.getCanonicalPath)\n+          .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+          .option(\"topic\", topic)\n+          .queryName(\"kafkaBadTopicStream\")\n+          .start()\n+        input.addData(\"1\", \"2\", \"3\", \"4\", \"5\")\n+        writer.processAllAvailable()\n+      }\n+      writer.stop()\n+      assert(ex.getMessage.toLowerCase.contains(\"job aborted\"))\n+    }\n+  }\n+\n+  test(\"write batch to kafka\") {\n+    val topic = newTopic()\n+    testUtils.createTopic(topic)\n+    val df = spark\n+      .sparkContext\n+      .parallelize(Seq(\"1\", \"2\", \"3\", \"4\", \"5\"))\n+      .map(v => (topic, v))\n+      .toDF(\"topic\", \"value\")\n+\n+    df.write\n+      .format(\"kafka\")\n+      .option(\"kafka.bootstrap.servers\", testUtils.brokerAddress)\n+      .option(\"topic\", topic)\n+      .save()\n+  }\n+\n+  test(\"write batch with null topic field value, and no topic option\") {\n+    val df = spark","path":"external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala","position":340,"original_position":340,"commit_id":"e6b6dc1035588dfdb8d4a16e2004ad3e0c838adb","original_commit_id":"e6b6dc1035588dfdb8d4a16e2004ad3e0c838adb","user":{"login":"zsxwing","id":1000778,"avatar_url":"https://avatars.githubusercontent.com/u/1000778?v=3","gravatar_id":"","url":"https://api.github.com/users/zsxwing","html_url":"https://github.com/zsxwing","followers_url":"https://api.github.com/users/zsxwing/followers","following_url":"https://api.github.com/users/zsxwing/following{/other_user}","gists_url":"https://api.github.com/users/zsxwing/gists{/gist_id}","starred_url":"https://api.github.com/users/zsxwing/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zsxwing/subscriptions","organizations_url":"https://api.github.com/users/zsxwing/orgs","repos_url":"https://api.github.com/users/zsxwing/repos","events_url":"https://api.github.com/users/zsxwing/events{/privacy}","received_events_url":"https://api.github.com/users/zsxwing/received_events","type":"User","site_admin":false},"body":"nit: `val df = Seq[(String, String)](null -> \"1\").toDF(\"topic\", \"value\")`","created_at":"2017-02-24T23:23:05Z","updated_at":"2017-02-24T23:30:45Z","html_url":"https://github.com/apache/spark/pull/17043#discussion_r103054126","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/17043","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/103054126"},"html":{"href":"https://github.com/apache/spark/pull/17043#discussion_r103054126"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/17043"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/17043","id":107720817,"html_url":"https://github.com/apache/spark/pull/17043","diff_url":"https://github.com/apache/spark/pull/17043.diff","patch_url":"https://github.com/apache/spark/pull/17043.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/17043","number":17043,"state":"open","locked":false,"title":"[SPARK-19719][SS][WIP] Kafka writer for both structured streaming and batch queires","user":{"login":"tcondie","id":1696558,"avatar_url":"https://avatars.githubusercontent.com/u/1696558?v=3","gravatar_id":"","url":"https://api.github.com/users/tcondie","html_url":"https://github.com/tcondie","followers_url":"https://api.github.com/users/tcondie/followers","following_url":"https://api.github.com/users/tcondie/following{/other_user}","gists_url":"https://api.github.com/users/tcondie/gists{/gist_id}","starred_url":"https://api.github.com/users/tcondie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tcondie/subscriptions","organizations_url":"https://api.github.com/users/tcondie/orgs","repos_url":"https://api.github.com/users/tcondie/repos","events_url":"https://api.github.com/users/tcondie/events{/privacy}","received_events_url":"https://api.github.com/users/tcondie/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nAdd a new Kafka Sink and Kafka Relation for writing streaming and batch queries, respectively, to Apache Kafka. \r\n\r\n@tdas @zsxwing \r\n\r\n## How was this patch tested?\r\n\r\nKafkaSinkSuite.scala contains tests for writing both streaming and batch queries to Kafka.\r\n\r\nPlease review http://spark.apache.org/contributing.html before opening a pull request.\r\n","created_at":"2017-02-23T21:50:38Z","updated_at":"2017-02-24T23:30:45Z","closed_at":null,"merged_at":null,"merge_commit_sha":"52d713a80dda0e4e475ee689a075d053bd836907","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/17043/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/17043/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/17043/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/e6b6dc1035588dfdb8d4a16e2004ad3e0c838adb","head":{"label":"tcondie:kafka-writer","ref":"kafka-writer","sha":"e6b6dc1035588dfdb8d4a16e2004ad3e0c838adb","user":{"login":"tcondie","id":1696558,"avatar_url":"https://avatars.githubusercontent.com/u/1696558?v=3","gravatar_id":"","url":"https://api.github.com/users/tcondie","html_url":"https://github.com/tcondie","followers_url":"https://api.github.com/users/tcondie/followers","following_url":"https://api.github.com/users/tcondie/following{/other_user}","gists_url":"https://api.github.com/users/tcondie/gists{/gist_id}","starred_url":"https://api.github.com/users/tcondie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tcondie/subscriptions","organizations_url":"https://api.github.com/users/tcondie/orgs","repos_url":"https://api.github.com/users/tcondie/repos","events_url":"https://api.github.com/users/tcondie/events{/privacy}","received_events_url":"https://api.github.com/users/tcondie/received_events","type":"User","site_admin":false},"repo":{"id":71503469,"name":"spark","full_name":"tcondie/spark","owner":{"login":"tcondie","id":1696558,"avatar_url":"https://avatars.githubusercontent.com/u/1696558?v=3","gravatar_id":"","url":"https://api.github.com/users/tcondie","html_url":"https://github.com/tcondie","followers_url":"https://api.github.com/users/tcondie/followers","following_url":"https://api.github.com/users/tcondie/following{/other_user}","gists_url":"https://api.github.com/users/tcondie/gists{/gist_id}","starred_url":"https://api.github.com/users/tcondie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tcondie/subscriptions","organizations_url":"https://api.github.com/users/tcondie/orgs","repos_url":"https://api.github.com/users/tcondie/repos","events_url":"https://api.github.com/users/tcondie/events{/privacy}","received_events_url":"https://api.github.com/users/tcondie/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tcondie/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/tcondie/spark","forks_url":"https://api.github.com/repos/tcondie/spark/forks","keys_url":"https://api.github.com/repos/tcondie/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tcondie/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tcondie/spark/teams","hooks_url":"https://api.github.com/repos/tcondie/spark/hooks","issue_events_url":"https://api.github.com/repos/tcondie/spark/issues/events{/number}","events_url":"https://api.github.com/repos/tcondie/spark/events","assignees_url":"https://api.github.com/repos/tcondie/spark/assignees{/user}","branches_url":"https://api.github.com/repos/tcondie/spark/branches{/branch}","tags_url":"https://api.github.com/repos/tcondie/spark/tags","blobs_url":"https://api.github.com/repos/tcondie/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tcondie/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tcondie/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/tcondie/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tcondie/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/tcondie/spark/languages","stargazers_url":"https://api.github.com/repos/tcondie/spark/stargazers","contributors_url":"https://api.github.com/repos/tcondie/spark/contributors","subscribers_url":"https://api.github.com/repos/tcondie/spark/subscribers","subscription_url":"https://api.github.com/repos/tcondie/spark/subscription","commits_url":"https://api.github.com/repos/tcondie/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/tcondie/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/tcondie/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/tcondie/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/tcondie/spark/contents/{+path}","compare_url":"https://api.github.com/repos/tcondie/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tcondie/spark/merges","archive_url":"https://api.github.com/repos/tcondie/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tcondie/spark/downloads","issues_url":"https://api.github.com/repos/tcondie/spark/issues{/number}","pulls_url":"https://api.github.com/repos/tcondie/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/tcondie/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/tcondie/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tcondie/spark/labels{/name}","releases_url":"https://api.github.com/repos/tcondie/spark/releases{/id}","deployments_url":"https://api.github.com/repos/tcondie/spark/deployments","created_at":"2016-10-20T20:51:29Z","updated_at":"2016-10-20T20:51:53Z","pushed_at":"2017-02-24T20:16:57Z","git_url":"git://github.com/tcondie/spark.git","ssh_url":"git@github.com:tcondie/spark.git","clone_url":"https://github.com/tcondie/spark.git","svn_url":"https://github.com/tcondie/spark","homepage":null,"size":260843,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"f87a6a59af5037c28d8b3c801c586b347f0ae10c","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2017-02-24T21:13:41Z","pushed_at":"2017-02-24T23:30:32Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":263792,"stargazers_count":11804,"watchers_count":11804,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":11082,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":451,"forks":11082,"open_issues":451,"watchers":11804,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/17043"},"html":{"href":"https://github.com/apache/spark/pull/17043"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/17043"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/17043/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/17043/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/17043/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/e6b6dc1035588dfdb8d4a16e2004ad3e0c838adb"}}}}
{"actor":{"display_login":"zsxwing"}}