{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/155429024","pull_request_review_id":81745757,"id":155429024,"diff_hunk":"@@ -0,0 +1,368 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.execution.datasources.orc\n+\n+import java.nio.charset.StandardCharsets\n+import java.sql.{Date, Timestamp}\n+\n+import scala.collection.JavaConverters._\n+\n+import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}\n+\n+import org.apache.spark.sql.{Column, DataFrame}\n+import org.apache.spark.sql.catalyst.dsl.expressions._\n+import org.apache.spark.sql.catalyst.expressions._\n+import org.apache.spark.sql.catalyst.planning.PhysicalOperation\n+import org.apache.spark.sql.execution.datasources.{DataSourceStrategy, HadoopFsRelation, LogicalRelation}\n+import org.apache.spark.sql.test.SharedSQLContext\n+import org.apache.spark.sql.types._\n+\n+/**\n+ * A test suite that tests Apache ORC filter API based filter pushdown optimization.\n+ */\n+class OrcFilterSuite extends OrcTest with SharedSQLContext {\n+\n+  private def checkFilterPredicate(\n+      df: DataFrame,\n+      predicate: Predicate,\n+      checker: (SearchArgument) => Unit): Unit = {\n+    val output = predicate.collect { case a: Attribute => a }.distinct\n+    val query = df\n+      .select(output.map(e => Column(e)): _*)\n+      .where(Column(predicate))\n+\n+    var maybeRelation: Option[HadoopFsRelation] = None\n+    val maybeAnalyzedPredicate = query.queryExecution.optimizedPlan.collect {\n+      case PhysicalOperation(_, filters, LogicalRelation(orcRelation: HadoopFsRelation, _, _, _)) =>\n+        maybeRelation = Some(orcRelation)\n+        filters\n+    }.flatten.reduceLeftOption(_ && _)\n+    assert(maybeAnalyzedPredicate.isDefined, \"No filter is analyzed from the given query\")\n+\n+    val (_, selectedFilters, _) =\n+      DataSourceStrategy.selectFilters(maybeRelation.get, maybeAnalyzedPredicate.toSeq)\n+    assert(selectedFilters.nonEmpty, \"No filter is pushed down\")\n+\n+    val maybeFilter = OrcFilters.createFilter(query.schema, selectedFilters)\n+    assert(maybeFilter.isDefined, s\"Couldn't generate filter predicate for $selectedFilters\")\n+    checker(maybeFilter.get)\n+  }\n+\n+  private def checkFilterPredicate\n+      (predicate: Predicate, filterOperator: PredicateLeaf.Operator)\n+      (implicit df: DataFrame): Unit = {\n+    def checkComparisonOperator(filter: SearchArgument) = {\n+      val operator = filter.getLeaves.asScala\n+      assert(operator.map(_.getOperator).contains(filterOperator))\n+    }\n+    checkFilterPredicate(df, predicate, checkComparisonOperator)\n+  }\n+\n+  private def checkFilterPredicate\n+      (predicate: Predicate, stringExpr: String)\n+      (implicit df: DataFrame): Unit = {\n+    def checkLogicalOperator(filter: SearchArgument) = {\n+      assert(filter.toString == stringExpr)\n+    }\n+    checkFilterPredicate(df, predicate, checkLogicalOperator)\n+  }\n+\n+  private def checkNoFilterPredicate\n+      (predicate: Predicate)\n+      (implicit df: DataFrame): Unit = {\n+    val output = predicate.collect { case a: Attribute => a }.distinct\n+    val query = df\n+      .select(output.map(e => Column(e)): _*)\n+      .where(Column(predicate))\n+\n+    var maybeRelation: Option[HadoopFsRelation] = None\n+    val maybeAnalyzedPredicate = query.queryExecution.optimizedPlan.collect {\n+      case PhysicalOperation(_, filters, LogicalRelation(orcRelation: HadoopFsRelation, _, _, _)) =>\n+        maybeRelation = Some(orcRelation)\n+        filters\n+    }.flatten.reduceLeftOption(_ && _)\n+    assert(maybeAnalyzedPredicate.isDefined, \"No filter is analyzed from the given query\")\n+\n+    val (_, selectedFilters, _) =\n+      DataSourceStrategy.selectFilters(maybeRelation.get, maybeAnalyzedPredicate.toSeq)\n+    assert(selectedFilters.nonEmpty, \"No filter is pushed down\")\n+\n+    val maybeFilter = OrcFilters.createFilter(query.schema, selectedFilters)\n+    assert(maybeFilter.isEmpty, s\"Could generate filter predicate for $selectedFilters\")\n+  }\n+\n+  test(\"filter pushdown - integer\") {\n+    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>\n+      checkFilterPredicate('_1.isNull, PredicateLeaf.Operator.IS_NULL)\n+\n+      checkFilterPredicate('_1 === 1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate('_1 <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+\n+      checkFilterPredicate('_1 < 2, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate('_1 > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 >= 4, PredicateLeaf.Operator.LESS_THAN)\n+\n+      checkFilterPredicate(Literal(1) === '_1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate(Literal(1) <=> '_1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+      checkFilterPredicate(Literal(2) > '_1, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate(Literal(3) < '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(1) >= '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(4) <= '_1, PredicateLeaf.Operator.LESS_THAN)\n+    }\n+  }\n+\n+  test(\"filter pushdown - long\") {\n+    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toLong)))) { implicit df =>\n+      checkFilterPredicate('_1.isNull, PredicateLeaf.Operator.IS_NULL)\n+\n+      checkFilterPredicate('_1 === 1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate('_1 <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+\n+      checkFilterPredicate('_1 < 2, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate('_1 > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 >= 4, PredicateLeaf.Operator.LESS_THAN)\n+\n+      checkFilterPredicate(Literal(1) === '_1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate(Literal(1) <=> '_1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+      checkFilterPredicate(Literal(2) > '_1, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate(Literal(3) < '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(1) >= '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(4) <= '_1, PredicateLeaf.Operator.LESS_THAN)\n+    }\n+  }\n+\n+  test(\"filter pushdown - float\") {\n+    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toFloat)))) { implicit df =>\n+      checkFilterPredicate('_1.isNull, PredicateLeaf.Operator.IS_NULL)\n+\n+      checkFilterPredicate('_1 === 1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate('_1 <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+\n+      checkFilterPredicate('_1 < 2, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate('_1 > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 >= 4, PredicateLeaf.Operator.LESS_THAN)\n+\n+      checkFilterPredicate(Literal(1) === '_1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate(Literal(1) <=> '_1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+      checkFilterPredicate(Literal(2) > '_1, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate(Literal(3) < '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(1) >= '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(4) <= '_1, PredicateLeaf.Operator.LESS_THAN)\n+    }\n+  }\n+\n+  test(\"filter pushdown - double\") {\n+    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i.toDouble)))) { implicit df =>\n+      checkFilterPredicate('_1.isNull, PredicateLeaf.Operator.IS_NULL)\n+\n+      checkFilterPredicate('_1 === 1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate('_1 <=> 1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+\n+      checkFilterPredicate('_1 < 2, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate('_1 > 3, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 <= 1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 >= 4, PredicateLeaf.Operator.LESS_THAN)\n+\n+      checkFilterPredicate(Literal(1) === '_1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate(Literal(1) <=> '_1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+      checkFilterPredicate(Literal(2) > '_1, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate(Literal(3) < '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(1) >= '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(4) <= '_1, PredicateLeaf.Operator.LESS_THAN)\n+    }\n+  }\n+\n+  test(\"filter pushdown - string\") {\n+    withOrcDataFrame((1 to 4).map(i => Tuple1(i.toString))) { implicit df =>\n+      checkFilterPredicate('_1.isNull, PredicateLeaf.Operator.IS_NULL)\n+\n+      checkFilterPredicate('_1 === \"1\", PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate('_1 <=> \"1\", PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+\n+      checkFilterPredicate('_1 < \"2\", PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate('_1 > \"3\", PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 <= \"1\", PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 >= \"4\", PredicateLeaf.Operator.LESS_THAN)\n+\n+      checkFilterPredicate(Literal(\"1\") === '_1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate(Literal(\"1\") <=> '_1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+      checkFilterPredicate(Literal(\"2\") > '_1, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate(Literal(\"3\") < '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(\"1\") >= '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(\"4\") <= '_1, PredicateLeaf.Operator.LESS_THAN)\n+    }\n+  }\n+\n+  test(\"filter pushdown - boolean\") {\n+    withOrcDataFrame((true :: false :: Nil).map(b => Tuple1.apply(Option(b)))) { implicit df =>\n+      checkFilterPredicate('_1.isNull, PredicateLeaf.Operator.IS_NULL)\n+\n+      checkFilterPredicate('_1 === true, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate('_1 <=> true, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+\n+      checkFilterPredicate('_1 < true, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate('_1 > false, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 <= false, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 >= false, PredicateLeaf.Operator.LESS_THAN)\n+\n+      checkFilterPredicate(Literal(false) === '_1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate(Literal(false) <=> '_1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+      checkFilterPredicate(Literal(false) > '_1, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate(Literal(true) < '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(true) >= '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(true) <= '_1, PredicateLeaf.Operator.LESS_THAN)\n+    }\n+  }\n+\n+  test(\"filter pushdown - decimal\") {\n+    withOrcDataFrame((1 to 4).map(i => Tuple1.apply(BigDecimal.valueOf(i)))) { implicit df =>\n+      checkFilterPredicate('_1.isNull, PredicateLeaf.Operator.IS_NULL)\n+\n+      checkFilterPredicate('_1 === BigDecimal.valueOf(1), PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate('_1 <=> BigDecimal.valueOf(1), PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+\n+      checkFilterPredicate('_1 < BigDecimal.valueOf(2), PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate('_1 > BigDecimal.valueOf(3), PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 <= BigDecimal.valueOf(1), PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 >= BigDecimal.valueOf(4), PredicateLeaf.Operator.LESS_THAN)\n+\n+      checkFilterPredicate(\n+        Literal(BigDecimal.valueOf(1)) === '_1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate(\n+        Literal(BigDecimal.valueOf(1)) <=> '_1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+      checkFilterPredicate(\n+        Literal(BigDecimal.valueOf(2)) > '_1, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate(\n+        Literal(BigDecimal.valueOf(3)) < '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(\n+        Literal(BigDecimal.valueOf(1)) >= '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(\n+        Literal(BigDecimal.valueOf(4)) <= '_1, PredicateLeaf.Operator.LESS_THAN)\n+    }\n+  }\n+\n+  test(\"filter pushdown - timestamp\") {\n+    val timeString = \"2015-08-20 14:57:00\"\n+    val timestamps = (1 to 4).map { i =>\n+      val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600\n+      new Timestamp(milliseconds)\n+    }\n+    withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>\n+      checkFilterPredicate('_1.isNull, PredicateLeaf.Operator.IS_NULL)\n+\n+      checkFilterPredicate('_1 === timestamps(0), PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate('_1 <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+\n+      checkFilterPredicate('_1 < timestamps(1), PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate('_1 > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate('_1 >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)\n+\n+      checkFilterPredicate(Literal(timestamps(0)) === '_1, PredicateLeaf.Operator.EQUALS)\n+      checkFilterPredicate(Literal(timestamps(0)) <=> '_1, PredicateLeaf.Operator.NULL_SAFE_EQUALS)\n+      checkFilterPredicate(Literal(timestamps(1)) > '_1, PredicateLeaf.Operator.LESS_THAN)\n+      checkFilterPredicate(Literal(timestamps(2)) < '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(timestamps(0)) >= '_1, PredicateLeaf.Operator.LESS_THAN_EQUALS)\n+      checkFilterPredicate(Literal(timestamps(3)) <= '_1, PredicateLeaf.Operator.LESS_THAN)\n+    }\n+  }\n+\n+  test(\"filter pushdown - combinations with logical operators\") {\n+    withOrcDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>\n+      // Because `ExpressionTree` is not accessible at Hive 1.2.x, this should be checked","path":"sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala","position":290,"original_position":290,"commit_id":"fcb2ccb64813f0fc37e3321c006937fb9cb870e4","original_commit_id":"fcb2ccb64813f0fc37e3321c006937fb9cb870e4","user":{"login":"HyukjinKwon","id":6477701,"avatar_url":"https://avatars0.githubusercontent.com/u/6477701?v=4","gravatar_id":"","url":"https://api.github.com/users/HyukjinKwon","html_url":"https://github.com/HyukjinKwon","followers_url":"https://api.github.com/users/HyukjinKwon/followers","following_url":"https://api.github.com/users/HyukjinKwon/following{/other_user}","gists_url":"https://api.github.com/users/HyukjinKwon/gists{/gist_id}","starred_url":"https://api.github.com/users/HyukjinKwon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HyukjinKwon/subscriptions","organizations_url":"https://api.github.com/users/HyukjinKwon/orgs","repos_url":"https://api.github.com/users/HyukjinKwon/repos","events_url":"https://api.github.com/users/HyukjinKwon/events{/privacy}","received_events_url":"https://api.github.com/users/HyukjinKwon/received_events","type":"User","site_admin":false},"body":"Yup.","created_at":"2017-12-07T04:35:45Z","updated_at":"2017-12-07T04:35:45Z","html_url":"https://github.com/apache/spark/pull/19882#discussion_r155429024","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/19882","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/155429024"},"html":{"href":"https://github.com/apache/spark/pull/19882#discussion_r155429024"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/19882"}},"in_reply_to_id":155410507},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/19882","id":156312344,"html_url":"https://github.com/apache/spark/pull/19882","diff_url":"https://github.com/apache/spark/pull/19882.diff","patch_url":"https://github.com/apache/spark/pull/19882.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/19882","number":19882,"state":"open","locked":false,"title":"[SPARK-22672][SQL][TEST] Refactor ORC Tests","user":{"login":"dongjoon-hyun","id":9700541,"avatar_url":"https://avatars2.githubusercontent.com/u/9700541?v=4","gravatar_id":"","url":"https://api.github.com/users/dongjoon-hyun","html_url":"https://github.com/dongjoon-hyun","followers_url":"https://api.github.com/users/dongjoon-hyun/followers","following_url":"https://api.github.com/users/dongjoon-hyun/following{/other_user}","gists_url":"https://api.github.com/users/dongjoon-hyun/gists{/gist_id}","starred_url":"https://api.github.com/users/dongjoon-hyun/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dongjoon-hyun/subscriptions","organizations_url":"https://api.github.com/users/dongjoon-hyun/orgs","repos_url":"https://api.github.com/users/dongjoon-hyun/repos","events_url":"https://api.github.com/users/dongjoon-hyun/events{/privacy}","received_events_url":"https://api.github.com/users/dongjoon-hyun/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nSince SPARK-20682, we have two `OrcFileFormat`s. This PR refactors ORC tests with three principles (with a few exceptions)\r\n1. Move test suite into `sql/core`.\r\n2. Create `HiveXXX` test suite in `sql/hive` by reusing `sql/core` test suite.\r\n3. `OrcTest` will provide common helper functions and `def format: String`.\r\n\r\n**Test Suites**\r\n\r\nOrcFileFormat in `sql/core` | OrcFileFormat in `sql/hive`\r\n-- | --\r\nOrcFilterSuite | HiveOrcFilterSuite\r\nOrcHadoopFsRelationSuite | HiveOrcHadoopFsRelationSuite\r\nOrcPartitionDiscoverySuite | HiveOrcPartitionDiscoverySuite\r\nOrcQuerySuite | HiveOrcQuerySuite\r\nOrcSourceSuite | HiveOrcSourceSuite\r\n\r\n**Hierarchy**\r\n```\r\nOrcTest\r\n    -> OrcSuite\r\n        -> OrcSourceSuite\r\n        -> OrcHiveSourceSuite\r\n    -> OrcQueryTest\r\n        -> OrcQuerySuite\r\n        -> HiveOrcQuerySuite\r\n    -> OrcPartitionDiscoveryTest\r\n        -> OrcPartitionDiscoverySuite\r\n        -> HiveOrcPartitionDiscoverySuite\r\n    -> OrcFilterSuite\r\n    -> HiveOrcFilterSuite\r\n\r\nHadoopFsRelationTest\r\n    -> OrcHadoopFsRelationSuite\r\n        -> HiveOrcHadoopFsRelationSuite\r\n```\r\n\r\nPlease note the followings.\r\n- Unlike the other test suites, `OrcHadoopFsRelationSuite` doesn't inherit `OrcTest`. It is inside `sql/hive` like `ParquetHadoopFsRelationSuite` due to the dependencies and follows the existing convention to use `val dataSourceName: String`\r\n- `OrcFilterSuite` and `HiveOrcFilterSuite` cannot reuse test cases due to the different function signatures using Hive 1.2.1 ORC classes and Apache ORC 1.4.1 classes.\r\n\r\n## How was this patch tested?\r\n\r\nPass the Jenkins tests with reorganized test suites.\r\n","created_at":"2017-12-04T21:35:15Z","updated_at":"2017-12-07T04:35:45Z","closed_at":null,"merged_at":null,"merge_commit_sha":"54bdd57afda187a9e77e114fcb1177eed7c5b0bb","assignee":null,"assignees":[],"requested_reviewers":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/19882/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/19882/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/19882/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/fcb2ccb64813f0fc37e3321c006937fb9cb870e4","head":{"label":"dongjoon-hyun:SPARK-22672","ref":"SPARK-22672","sha":"fcb2ccb64813f0fc37e3321c006937fb9cb870e4","user":{"login":"dongjoon-hyun","id":9700541,"avatar_url":"https://avatars2.githubusercontent.com/u/9700541?v=4","gravatar_id":"","url":"https://api.github.com/users/dongjoon-hyun","html_url":"https://github.com/dongjoon-hyun","followers_url":"https://api.github.com/users/dongjoon-hyun/followers","following_url":"https://api.github.com/users/dongjoon-hyun/following{/other_user}","gists_url":"https://api.github.com/users/dongjoon-hyun/gists{/gist_id}","starred_url":"https://api.github.com/users/dongjoon-hyun/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dongjoon-hyun/subscriptions","organizations_url":"https://api.github.com/users/dongjoon-hyun/orgs","repos_url":"https://api.github.com/users/dongjoon-hyun/repos","events_url":"https://api.github.com/users/dongjoon-hyun/events{/privacy}","received_events_url":"https://api.github.com/users/dongjoon-hyun/received_events","type":"User","site_admin":false},"repo":{"id":28900400,"name":"spark","full_name":"dongjoon-hyun/spark","owner":{"login":"dongjoon-hyun","id":9700541,"avatar_url":"https://avatars2.githubusercontent.com/u/9700541?v=4","gravatar_id":"","url":"https://api.github.com/users/dongjoon-hyun","html_url":"https://github.com/dongjoon-hyun","followers_url":"https://api.github.com/users/dongjoon-hyun/followers","following_url":"https://api.github.com/users/dongjoon-hyun/following{/other_user}","gists_url":"https://api.github.com/users/dongjoon-hyun/gists{/gist_id}","starred_url":"https://api.github.com/users/dongjoon-hyun/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dongjoon-hyun/subscriptions","organizations_url":"https://api.github.com/users/dongjoon-hyun/orgs","repos_url":"https://api.github.com/users/dongjoon-hyun/repos","events_url":"https://api.github.com/users/dongjoon-hyun/events{/privacy}","received_events_url":"https://api.github.com/users/dongjoon-hyun/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/dongjoon-hyun/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/dongjoon-hyun/spark","forks_url":"https://api.github.com/repos/dongjoon-hyun/spark/forks","keys_url":"https://api.github.com/repos/dongjoon-hyun/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dongjoon-hyun/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dongjoon-hyun/spark/teams","hooks_url":"https://api.github.com/repos/dongjoon-hyun/spark/hooks","issue_events_url":"https://api.github.com/repos/dongjoon-hyun/spark/issues/events{/number}","events_url":"https://api.github.com/repos/dongjoon-hyun/spark/events","assignees_url":"https://api.github.com/repos/dongjoon-hyun/spark/assignees{/user}","branches_url":"https://api.github.com/repos/dongjoon-hyun/spark/branches{/branch}","tags_url":"https://api.github.com/repos/dongjoon-hyun/spark/tags","blobs_url":"https://api.github.com/repos/dongjoon-hyun/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dongjoon-hyun/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dongjoon-hyun/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/dongjoon-hyun/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dongjoon-hyun/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/dongjoon-hyun/spark/languages","stargazers_url":"https://api.github.com/repos/dongjoon-hyun/spark/stargazers","contributors_url":"https://api.github.com/repos/dongjoon-hyun/spark/contributors","subscribers_url":"https://api.github.com/repos/dongjoon-hyun/spark/subscribers","subscription_url":"https://api.github.com/repos/dongjoon-hyun/spark/subscription","commits_url":"https://api.github.com/repos/dongjoon-hyun/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/dongjoon-hyun/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/dongjoon-hyun/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/dongjoon-hyun/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/dongjoon-hyun/spark/contents/{+path}","compare_url":"https://api.github.com/repos/dongjoon-hyun/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dongjoon-hyun/spark/merges","archive_url":"https://api.github.com/repos/dongjoon-hyun/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dongjoon-hyun/spark/downloads","issues_url":"https://api.github.com/repos/dongjoon-hyun/spark/issues{/number}","pulls_url":"https://api.github.com/repos/dongjoon-hyun/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/dongjoon-hyun/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/dongjoon-hyun/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dongjoon-hyun/spark/labels{/name}","releases_url":"https://api.github.com/repos/dongjoon-hyun/spark/releases{/id}","deployments_url":"https://api.github.com/repos/dongjoon-hyun/spark/deployments","created_at":"2015-01-07T05:46:34Z","updated_at":"2017-05-03T18:08:51Z","pushed_at":"2017-12-07T03:07:29Z","git_url":"git://github.com/dongjoon-hyun/spark.git","ssh_url":"git@github.com:dongjoon-hyun/spark.git","clone_url":"https://github.com/dongjoon-hyun/spark.git","svn_url":"https://github.com/dongjoon-hyun/spark","homepage":null,"size":258662,"stargazers_count":1,"watchers_count":1,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":1,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"f110a7f884cb09f01a20462038328ddc5662b46f","user":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2017-12-07T02:58:37Z","pushed_at":"2017-12-07T01:42:53Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":267385,"stargazers_count":15374,"watchers_count":15374,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":14251,"mirror_url":null,"archived":false,"open_issues_count":398,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":14251,"open_issues":398,"watchers":15374,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/19882"},"html":{"href":"https://github.com/apache/spark/pull/19882"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/19882"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/19882/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/19882/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/19882/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/fcb2ccb64813f0fc37e3321c006937fb9cb870e4"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"HyukjinKwon"}}