Feature/serverless v4 (#14)

* upgrade to serverless v4, upgrade databricks cli and uv

* explain databricks-connect usage

* change pinning

* rename dependencies group

* use all extras in cicd

* use proper connect version

* fix dbc tests

Author: frizzleqq

.github/workflows/ci.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Install uv and set the python version
       uses: astral-sh/setup-uv@v5
       with:
-        version: "0.8.0"
+        version: "0.9.16"
         python-version: ${{ matrix.python-version }}
         enable-cache: true
     - name: Setup Java JDK
@@ -32,7 +32,7 @@ jobs:
         java-version: 17
         distribution: "zulu"
     - name: Install the project
-      run: uv sync --locked --extra dev
+      run: uv sync --locked --all-extras
     - name: Run code checks
       run: uv run ruff check
     - name: Check code formatting
@@ -57,19 +57,19 @@ jobs:
     - name: Install uv and set the python version
       uses: astral-sh/setup-uv@v5
       with:
-        version: "0.8.0"
+        version: "0.9.16"
         python-version: ${{ matrix.python-version }}
         enable-cache: true
     - uses: databricks/setup-cli@main
       with:
-        version: 0.260.0
+        version: 0.279.0
     - name: Install the project
-      run: uv sync --locked --extra dev
+      run: uv sync --locked --all-extras
     - name: Install Databricks Connect
       run: |
         uv pip uninstall pyspark
-        uv pip install databricks-connect==16.3.5
+        uv pip install databricks-connect==17.2.*
     - name: Check Databricks CLI
       run: databricks current-user me
     - name: Run tests
-      run: uv run pytest -v
+      run: uv run --no-sync pytest -v

.github/workflows/deploy-test.yml

@@ -27,14 +27,14 @@ jobs:
     - name: Install uv and set the python version
       uses: astral-sh/setup-uv@v5
       with:
-        version: "0.8.0"
+        version: "0.9.16"
         python-version: ${{ matrix.python-version }}
         enable-cache: true
     - uses: databricks/setup-cli@main
       with:
-        version: 0.260.0
+        version: 0.279.0
     - name: Install the project
-      run: uv sync --locked --extra dev
+      run: uv sync --locked --all-extras
     - name: Check Databricks CLI
       run: databricks current-user me
     - name: Deploy Databricks Bundle

.github/workflows/release.yml

@@ -25,14 +25,14 @@ jobs:
     - name: Install uv and set the python version
       uses: astral-sh/setup-uv@v5
       with:
-        version: "0.8.0"
+        version: "0.9.16"
         python-version: ${{ matrix.python-version }}
         enable-cache: true
     - uses: databricks/setup-cli@main
       with:
-        version: 0.260.0
+        version: 0.279.0
     - name: Install the project
-      run: uv sync --locked --extra dev
+      run: uv sync --locked --all-extras
     - name: Check Databricks CLI
       run: databricks current-user me
     - name: Deploy Databricks Bundle

.github/workflows/validate-bundle.yml

@@ -26,14 +26,14 @@ jobs:
     - name: Install uv and set the python version
       uses: astral-sh/setup-uv@v5
       with:
-        version: "0.8.0"
+        version: "0.9.16"
         python-version: ${{ matrix.python-version }}
         enable-cache: true
     - uses: databricks/setup-cli@main
       with:
-        version: 0.260.0
+        version: 0.279.0
     - name: Install the project
-      run: uv sync --locked --extra dev
+      run: uv sync --locked --all-extras
     - name: Check Databricks CLI
       run: databricks current-user me
     - name: Validate Databricks Bundle

README.md

@@ -23,7 +23,7 @@ For this example we use a Databricks Free Edition workspace https://www.databric
 
 Groups and Service Principals are not necessary, but are used in this project to showcase handling permissions on resources such as catalogs or workflows.
 
-* **Serverless environment version 3**, which matches [Databricks Runtime 16.3](https://docs.databricks.com/aws/en/release-notes/serverless/#version-163)
+* **Serverless environment**: [Version 4](https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/four) which is similar to Databricks Runtime ~17.*
 * **Catalogs**: `lake_dev`, `lake_test` and `lake_prod`
 * **Service principals** (for CI/CD and Workflow runners)
   * `sp_etl_dev` (for dev and test) and `sp_etl_prod` (for prod)
@@ -45,9 +45,9 @@ A script exists set up the (Free) Workspace as described in [scripts/setup_works
 
 ### Setup environment
 
-Sync entire `uv` environment with dev dependencies:
+Sync entire `uv` environment with all optional dependency groups:
 ```bash
-uv sync --extra dev
+uv sync --all-extras
 ```
 
 > **Note:** we install Databricks Connect in a follow-up step
@@ -70,16 +70,24 @@ Install `databricks-connect` in active environment. This requires authentication
 
 ```bash
 uv pip uninstall pyspark
-uv pip install databricks-connect==16.3.5
+uv pip install databricks-connect==17.2.*
 ```
-> **Note:** For Databricks Runtime 16.3
+
+**Option 2: Run with temporary dependency**
+```bash
+uv run --with databricks-connect==17.2.* pytest
+```
+
+> **Note:** For Databricks Runtime Serverless v4
+
 
 See https://docs.databricks.com/aws/en/dev-tools/vscode-ext/ for using Databricks Connect extension in VS Code.
 
 ### Unit-Tests
 
 ```bash
-uv run pytest -v
+# in case databricks-connect is installed, --no-sync prevents reinstalling pyspark
+uv run --no-sync pytest -v
 ```
 
 Based on whether Databricks Connect is enabled or not the Unit-Tests try to use a Databricks Cluster or start a local Spark session with Delta support.

databricks.yml

@@ -9,7 +9,7 @@ variables:
     description: The name of the catalog to use for this bundle.
   serverless_environment_version:
     description: The serverless environment to use for this bundle.
-    default: "3"
+    default: "4"
   dbt_sql_warehouse_id:
     description: The ID of the SQL warehouse to use for dbt tasks.
 

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["uv_build>=0.8.0,<0.9"]
+requires = ["uv_build>=0.8.0,<0.10"]
 build-backend = "uv_build"
 
 [project]
@@ -13,22 +13,26 @@ authors = [
 requires-python = ">=3.12"
 urls.homepage = "https://github.com/frizzleqq/databricks-bundle-implementation"
 
-# Roughly based on Databricks runtime 16.3: https://docs.databricks.com/aws/en/release-notes/serverless/#version-163
-# Note that we do not include pre-installed libraries like "pydantic" in the runtime-dependencies here
+# Based on serverless environment 4: https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/four
+# Note that we do not include pre-installed libraries like "pydantic" in the runtime-dependencies in main dependencies
 dependencies = [
-    "databricks-sdk>=0.41, <0.48.0", # pinning from dbt-databricks
+    "databricks-sdk>=0.41, <0.68.0", # pinning from dbt-databricks
 ]
 
 [project.optional-dependencies]
+# Databricks runtime dependencies (preinstalled on cluster)
+databricks = [
+    "delta-spark==4.0.*",
+    "pydantic==2.10.6",
+    "pyspark==4.0.*",
+]
+# dbt dependencies
+dbt = [
+    "dbt-databricks==1.11.*",
+]
+# Development & Testing
 dev = [
-    # Runtime
-    "delta-spark>=3.3.0, <4.0.0",
-    "pydantic==2.8.2",
-    "pyspark>=3.5.0, <4.0.0",
-    # dbt
-    "dbt-databricks~=1.10.0",
-    # Tooling
-    "databricks-bundles~=0.260.0", # For Python-based Workflows
+    "databricks-bundles==0.279.*", # For Python-based Workflows
     "mypy", # Type hints
     "pip", # Databricks extension needs it
     "pytest", # Unit testing

scripts/setup_workspace.py

@@ -1,8 +1,8 @@
 # /// script
 # requires-python = ">=3.12"
 # dependencies = [
-#   "databricks-sdk==0.61.0",
-#   "databricks-connect~=16.3.0",
+#   "databricks-sdk>=0.41, <0.68.0",
+#   "databricks-connect==17.2.*",
 # ]
 # ///
 """Sets up the Databricks Workspace with necessary catalogs, groups and service principals.