Skip to content

Commit 554d678

Browse files
heyihongHyukjinKwon
authored andcommitted
[SPARK-51712][SQL] Swallow non-fatal Throwables when resolving tables/views in spark.catalog.listTables()
### What changes were proposed in this pull request? Swallow non-fatal Throwables when resolving tables/views in spark.catalog.listTables() and return tables/views with partial results instead ### Why are the changes needed? Prevent listTables from breaking easily due to a broken table/view. ### Does this PR introduce _any_ user-facing change? No, but listTables won't break easily. ### How was this patch tested? Existing tests (e.g. HiveDDLSuite) ### Was this patch authored or co-authored using generative AI tooling? No Closes #50515 from heyihong/SPARK-51712. Authored-by: Yihong He <heyihong.cn@gmail.com> Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
1 parent 54288f8 commit 554d678

File tree

1 file changed

+34
-22
lines changed
  • sql/core/src/main/scala/org/apache/spark/sql/classic

1 file changed

+34
-22
lines changed

sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala

Lines changed: 34 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ package org.apache.spark.sql.classic
2020
import scala.reflect.runtime.universe.TypeTag
2121
import scala.util.control.NonFatal
2222

23+
import org.apache.spark.internal.{Logging, MDC}
24+
import org.apache.spark.internal.LogKeys.{CATALOG_NAME, DATABASE_NAME, TABLE_NAME}
2325
import org.apache.spark.sql.AnalysisException
2426
import org.apache.spark.sql.catalog
2527
import org.apache.spark.sql.catalog.{CatalogMetadata, Column, Database, Function, Table}
@@ -48,7 +50,7 @@ import org.apache.spark.util.ArrayImplicits._
4850
/**
4951
* Internal implementation of the user-facing `Catalog`.
5052
*/
51-
class Catalog(sparkSession: SparkSession) extends catalog.Catalog {
53+
class Catalog(sparkSession: SparkSession) extends catalog.Catalog with Logging {
5254

5355
private def sessionCatalog: SessionCatalog = sparkSession.sessionState.catalog
5456

@@ -167,30 +169,40 @@ class Catalog(sparkSession: SparkSession) extends catalog.Catalog {
167169
private[sql] def resolveTable(row: InternalRow, catalogName: String): Option[Table] = {
168170
val tableName = row.getString(1)
169171
val namespaceName = row.getString(0)
170-
val isTemp = row.getBoolean(2)
172+
val isTempView = row.getBoolean(2)
173+
val ns = if (isTempView) {
174+
if (namespaceName.isEmpty) Nil else Seq(namespaceName)
175+
} else {
176+
parseIdent(namespaceName)
177+
}
178+
val nameParts = if (isTempView) {
179+
// Temp views do not belong to any catalog. We shouldn't prepend the catalog name here.
180+
ns :+ tableName
181+
} else {
182+
catalogName +: ns :+ tableName
183+
}
171184
try {
172-
if (isTemp) {
173-
// Temp views do not belong to any catalog. We shouldn't prepend the catalog name here.
174-
val ns = if (namespaceName.isEmpty) Nil else Seq(namespaceName)
175-
Some(makeTable(ns :+ tableName))
176-
} else {
177-
val ns = parseIdent(namespaceName)
178-
try {
179-
Some(makeTable(catalogName +: ns :+ tableName))
180-
} catch {
181-
case e: AnalysisException if e.getCondition == "UNSUPPORTED_FEATURE.HIVE_TABLE_TYPE" =>
182-
Some(new Table(
183-
name = tableName,
184-
catalog = catalogName,
185-
namespace = ns.toArray,
186-
description = null,
187-
tableType = null,
188-
isTemporary = false
189-
))
190-
}
191-
}
185+
Some(makeTable(nameParts))
192186
} catch {
193187
case e: AnalysisException if e.getCondition == "TABLE_OR_VIEW_NOT_FOUND" => None
188+
// Swallow non-fatal throwables when resolving a table or view and
189+
// return a table or view with partial results to
190+
// prevent listTables from breaking easily due to a broken table or view.
191+
case NonFatal(e) if !isTempView =>
192+
val table = new Table(
193+
name = tableName,
194+
catalog = catalogName,
195+
namespace = ns.toArray,
196+
description = null,
197+
tableType = null,
198+
isTemporary = false
199+
)
200+
logWarning(log"Unable to resolve the table or view [" +
201+
log"catalog=${MDC(CATALOG_NAME, table.catalog)}, " +
202+
log"database=${MDC(DATABASE_NAME, table.database)}, " +
203+
log"name=${MDC(TABLE_NAME, table.name)}" +
204+
log"]; partial results will be returned.", e)
205+
Some(table)
194206
}
195207
}
196208

0 commit comments

Comments
 (0)