Skip to content

Commit 6052121

Browse files
TongWei1105wangyum
authored andcommitted
[SPARK-49507][SQL] Fix the case issue after enabling metastorePartitionPruningFastFallback
### What changes were proposed in this pull request? This PR enhances the partition predicate handling in the `HiveShim` by ensuring that partition schema and predicates are properly transformed to lowercase. This change improves compatibility when generating partition predicates for filtering. How to reproduce: ``` CREATE TABLE t (ID BIGINT, DT STRING) USING parquet PARTITIONED BY (DT); set spark.sql.hive.metastorePartitionPruningFastFallback=true; select * from t where dt=20240820; ``` Error message: ``` org.apache.spark.sql.AnalysisException: Expected only partition pruning predicates: List(isnotnull(DT#21), (cast(DT#21 as bigint) = 20240820)). at org.apache.spark.sql.errors.QueryCompilationErrors$.nonPartitionPruningPredicatesNotExpectedError(QueryCompilationErrors.scala:2414) at org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils$.generatePartitionPredicateByFilter(ExternalCatalo ``` ### Why are the changes needed? Bug fix. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Unit test. ### Was this patch authored or co-authored using generative AI tooling? No Closes #47998 from TongWei1105/SPARK-49507. Authored-by: TongWei1105 <vvtwow@gmail.com> Signed-off-by: Yuming Wang <yumwang@ebay.com>
1 parent 6ba4771 commit 6052121

File tree

2 files changed

+14
-1
lines changed

2 files changed

+14
-1
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -417,8 +417,11 @@ private[client] class Shim_v2_0 extends Shim with Logging {
417417
try {
418418
val partitionSchema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(
419419
catalogTable.partitionSchema)
420+
val lowerCasePredicates = predicates.map(_.transform {
421+
case a: AttributeReference => a.withName(a.name.toLowerCase(Locale.ROOT))
422+
})
420423
val boundPredicate = ExternalCatalogUtils.generatePartitionPredicateByFilter(
421-
catalogTable, partitionSchema, predicates)
424+
catalogTable, partitionSchema, lowerCasePredicates)
422425

423426
def toRow(spec: TablePartitionSpec): InternalRow = {
424427
InternalRow.fromSeq(partitionSchema.map { field =>

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitionsSuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,16 @@ class PruneHiveTablePartitionsSuite extends PrunePartitionSuiteBase with TestHiv
177177
}
178178
}
179179

180+
test("SPARK-49507: Fix the case issue after enabling metastorePartitionPruningFastFallback") {
181+
withTable("t") {
182+
withSQLConf(SQLConf.HIVE_METASTORE_PARTITION_PRUNING_FAST_FALLBACK.key -> "true") {
183+
sql("CREATE TABLE t(ID BIGINT, DT STRING) USING PARQUET PARTITIONED BY (DT)")
184+
sql("INSERT INTO TABLE t SELECT 1, '20240820'")
185+
checkAnswer(sql("SELECT * FROM t WHERE dt=20240820"), Row(1, "20240820") :: Nil)
186+
}
187+
}
188+
}
189+
180190
protected def collectPartitionFiltersFn(): PartialFunction[SparkPlan, Seq[Expression]] = {
181191
case scan: HiveTableScanExec => scan.partitionPruningPred
182192
}

0 commit comments

Comments
 (0)