improve conftest

Author: frizzleqq

tests/conftest.py

@@ -2,7 +2,7 @@
 import tempfile
 import uuid
 from pathlib import Path
-from typing import Generator
+from typing import Generator, Optional
 
 import pytest
 from delta import configure_spark_with_delta_pip
@@ -24,7 +24,7 @@ def spark() -> Generator[SparkSession, None, None]:
         yield spark
     else:
         # If databricks-connect is not installed, we use use local Spark session
-        warehouse_dir = tempfile.TemporaryDirectory().name
+        warehouse_dir = tempfile.mkdtemp()
         _builder = (
             SparkSession.builder.master("local[*]")
             .config("spark.hive.metastore.warehouse.dir", Path(warehouse_dir).as_uri())
@@ -47,7 +47,7 @@ def spark() -> Generator[SparkSession, None, None]:
 
 
 @pytest.fixture(scope="session")
-def catalog_name() -> str:
+def catalog_name() -> Optional[str]:
     """Fixture to provide the catalog name for tests.
 
     In Databricks, we use the "lake_dev" catalog.
@@ -67,11 +67,15 @@ def create_schema(spark, catalog_name, request) -> Generator[str, None, None]:
     """
     module_name = request.module.__name__.split(".")[-1]  # Get just the module name without path
     schema_name = f"pytest_{module_name}_{uuid.uuid4().hex[:8]}"
+
     if catalog_name is not None:
         full_schema_name = f"{catalog_name}.{schema_name}"
+    else:
+        full_schema_name = schema_name
+
     spark.sql(f"CREATE SCHEMA IF NOT EXISTS {full_schema_name}")
     yield schema_name
-    spark.sql(f"DROP SCHEMA {full_schema_name} CASCADE")
+    spark.sql(f"DROP SCHEMA IF EXISTS {full_schema_name} CASCADE")
 
 
 @pytest.fixture(scope="function")