WIP - deploying without terraform

Author: denik

.github/workflows/push.yml

@@ -42,15 +42,24 @@ jobs:
           - macos-latest
           - ubuntu-latest
           - windows-latest
+        deployment:
+          - "terraform"
+          - "direct"
 
     steps:
       - name: Checkout repository and submodules
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+      - name: Create deployment-specific cache identifier
+        run: echo "${{ matrix.deployment }}" > deployment-type.txt
+
       - name: Setup Go
         uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version-file: go.mod
+          cache-dependency-path: |
+            go.sum
+            deployment-type.txt
 
       - name: Setup Python
         uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
@@ -72,11 +81,15 @@ jobs:
         # and would like to run the tests as fast as possible. We run it on schedule as well, because that is what
         # populates the cache and cache may include test results.
         if: ${{ github.event_name == 'pull_request' || github.event_name == 'schedule' }}
+        env:
+          DATABRICKS_CLI_DEPLOYMENT: ${{ matrix.deployment }}
         run: make test
 
       - name: Run tests with coverage
         # Still run 'make cover' on push to main and merge checks to make sure it does not get broken.
         if: ${{ github.event_name != 'pull_request' && github.event_name != 'schedule' }}
+        env:
+          DATABRICKS_CLI_DEPLOYMENT: ${{ matrix.deployment }}
         run: make cover
 
       - name: Analyze slow tests

acceptance/acceptance_test.go

@@ -36,6 +36,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const deploymentEnvVar = "DATABRICKS_CLI_DEPLOYMENT"
+
 var (
 	KeepTmp     bool
 	NoRepl      bool
@@ -214,6 +216,15 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 	totalDirs := 0
 	selectedDirs := 0
 
+	deploymentFilter, hasFilter := os.LookupEnv(deploymentEnvVar)
+	filterEnv := ""
+	if hasFilter {
+		filterEnv = deploymentEnvVar + "=" + deploymentFilter
+
+		// Clear it just to be sure, since it's going to be part of os.Environ() and we're going to add different value based on settings.
+		t.Setenv(deploymentEnvVar, "")
+	}
+
 	for _, dir := range testDirs {
 		totalDirs += 1
 
@@ -245,15 +256,15 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 				if len(expanded[0]) > 0 {
 					t.Logf("Running test with env %v", expanded[0])
 				}
-				runTest(t, dir, 0, coverDir, repls.Clone(), config, configPath, expanded[0], inprocessMode)
+				runTest(t, dir, 0, coverDir, repls.Clone(), config, configPath, expanded[0], inprocessMode, filterEnv)
 			} else {
 				for ind, envset := range expanded {
 					envname := strings.Join(envset, "/")
 					t.Run(envname, func(t *testing.T) {
 						if !inprocessMode {
 							t.Parallel()
 						}
-						runTest(t, dir, ind, coverDir, repls.Clone(), config, configPath, envset, inprocessMode)
+						runTest(t, dir, ind, coverDir, repls.Clone(), config, configPath, envset, inprocessMode, filterEnv)
 					})
 				}
 			}
@@ -354,6 +365,7 @@ func runTest(t *testing.T,
 	configPath string,
 	customEnv []string,
 	inprocessMode bool,
+	filterEnv string,
 ) {
 	if LogConfig {
 		configBytes, err := json.MarshalIndent(config, "", "  ")
@@ -485,6 +497,23 @@ func runTest(t *testing.T,
 		cmd.Env = addEnvVar(t, cmd.Env, &repls, key, value, config.EnvRepl, len(config.EnvMatrix[key]) > 1)
 	}
 
+	if filterEnv != "" {
+		filterEnvKey := strings.Split(filterEnv, "=")[0]
+		for ind := range cmd.Env {
+			// Search backwards, because the latest settings is what is actually applicable.
+			// For cases like DATABRICKS_CLI_DEPLOYMENT, there will be 2 instances: one copied from os.Environ,
+			// the other one coming from EnvMatrix.
+			envPair := cmd.Env[len(cmd.Env)-1-ind]
+			if strings.Split(envPair, "=")[0] == filterEnvKey {
+				if envPair == filterEnv {
+					break
+				} else {
+					t.Skipf("Skipping because test environment %s does not match requested env %s", envPair, filterEnv)
+				}
+			}
+		}
+	}
+
 	absDir, err := filepath.Abs(dir)
 	require.NoError(t, err)
 	cmd.Env = append(cmd.Env, "TESTDIR="+absDir)

acceptance/bin/read_id.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""
+Print selected attributes from terraform state.
+
+Usage: <section> <name> [attr...]
+"""
+
+import sys
+import os
+import json
+
+
+def print_resource_terraform(section, name):
+    resource_type = "databricks_" + section[:-1]
+    filename = ".databricks/bundle/default/terraform/terraform.tfstate"
+    raw = open(filename).read()
+    data = json.loads(raw)
+    found = 0
+    for r in data["resources"]:
+        r_type = r["type"]
+        r_name = r["name"]
+        if r_type != resource_type:
+            continue
+        if r_name != name:
+            continue
+        for inst in r["instances"]:
+            attribute_values = inst.get("attributes") or {}
+            print(attribute_values.get("id"))
+            return
+
+
+def print_resource_terranova(section, name):
+    filename = ".databricks/bundle/default/resources.json"
+    raw = open(filename).read()
+    data = json.loads(raw)
+    resources = data["resources"].get(section, {})
+    result = resources.get(name)
+    if result is None:
+        print(f"Resource {section=} {name=} not found. Available: {raw}")
+        return
+    print(result.get("__id__"))
+
+
+if os.environ.get("DATABRICKS_CLI_DEPLOYMENT") == "direct":
+    print_resource_terranova(*sys.argv[1:])
+else:
+    print_resource_terraform(*sys.argv[1:])

acceptance/bin/read_state.py

@@ -13,17 +13,15 @@
 def print_resource_terraform(section, name, *attrs):
     resource_type = "databricks_" + section[:-1]
     filename = ".databricks/bundle/default/terraform/terraform.tfstate"
-    data = json.load(open(filename))
-    available = []
+    raw = open(filename).read()
+    data = json.loads(raw)
     found = 0
     for r in data["resources"]:
         r_type = r["type"]
         r_name = r["name"]
         if r_type != resource_type:
-            available.append((r_type, r_name))
             continue
         if r_name != name:
-            available.append((r_type, r_name))
             continue
         for inst in r["instances"]:
             attribute_values = inst.get("attributes")
@@ -32,7 +30,25 @@ def print_resource_terraform(section, name, *attrs):
                 print(section, name, " ".join(values))
                 found += 1
     if not found:
-        print(f"Resource {(resource_type, name)} not found. Available: {available}")
+        print(f"Resource {section=} {name=} {resource_type=} not found. Available: {raw}")
 
 
-print_resource_terraform(*sys.argv[1:])
+def print_resource_terranova(section, name, *attrs):
+    filename = ".databricks/bundle/default/resources.json"
+    raw = open(filename).read()
+    data = json.loads(raw)
+    resources = data["resources"].get(section, {})
+    result = resources.get(name)
+    if result is None:
+        print(f"Resource {section=} {name=} not found. Available: {raw}")
+        return
+    state = result["state"]
+    state.setdefault("id", result.get("__id__"))
+    values = [f"{x}={state.get(x)!r}" for x in attrs]
+    print(section, name, " ".join(values))
+
+
+if os.environ.get("DATABRICKS_CLI_DEPLOYMENT") == "direct":
+    print_resource_terranova(*sys.argv[1:])
+else:
+    print_resource_terraform(*sys.argv[1:])

acceptance/bundle/artifacts/whl_dynamic/test.toml

@@ -1,3 +1,6 @@
+# Terraform sorts tasks
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 [[Repls]]
 Old = '\\\\'
 New = '/'

acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/test.toml

@@ -1,4 +1,5 @@
 BundleConfig.default_name = ""
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # need to sort tasks by key
 
 [[Repls]]
 Old = '\\'

acceptance/bundle/debug/test.toml

@@ -1,3 +1,6 @@
+# Debug output is naturally different. TODO: split debug tests in two: terraform and terranova
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 [[Repls]]
 # The keys are unsorted and also vary per OS
 Old = 'Environment variables for Terraform: ([A-Z_ ,]+) '

acceptance/bundle/deploy/dashboard/simple/test.toml

@@ -2,6 +2,9 @@ Local = true
 Cloud = true
 RequiresWarehouse = true
 
+# dashboards not implemented yet
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 Ignore = [
     "databricks.yml",
 ]

acceptance/bundle/deploy/fail-on-active-runs/test.toml

@@ -1,5 +1,8 @@
 RecordRequests = true
 
+# --fail-on-active-runs not implemented yet
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 [[Server]]
 Pattern = "GET /api/2.2/jobs/runs/list"
 Response.Body = '''

acceptance/bundle/deploy/jobs/double-underscore-keys/test.toml

@@ -1,6 +1,8 @@
 Local = true
 Cloud = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # "bundle destroy"
+
 Ignore = [
     "databricks.yml",
 ]

acceptance/bundle/deploy/jobs/fail-on-active-runs/test.toml

@@ -1,6 +1,8 @@
 Local = true
 Cloud = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 Ignore = [
     "databricks.yml",
 ]

acceptance/bundle/deploy/pipeline/auto-approve/test.toml

@@ -1,6 +1,8 @@
 Local = true
 Cloud = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 Ignore = [
     "databricks.yml"
 ]

acceptance/bundle/deploy/pipeline/recreate/test.toml

@@ -2,6 +2,8 @@ Local = true
 Cloud = true
 RequiresUnityCatalog = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 Ignore = [
     "databricks.yml"
 ]

acceptance/bundle/deploy/schema/auto-approve/test.toml

@@ -2,6 +2,8 @@ Local = true
 Cloud = true
 RequiresUnityCatalog = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 Ignore = [
     "databricks.yml",
     "test-file-*.txt",

acceptance/bundle/deploy/secret-scope/test.toml

@@ -1,6 +1,8 @@
 Cloud = true
 Local = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 Ignore = [
     "databricks.yml",
 ]

acceptance/bundle/deployment/bind/test.toml

@@ -1,2 +1,4 @@
 BundleConfig.default_name.bundle.name = "test-bundle-$UNIQUE_NAME"
 BundleConfigTarget = "databricks.yml.tmpl"
+
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]

acceptance/bundle/python/restricted-execution/test.toml

@@ -0,0 +1,2 @@
+# "bundle summary" is not implemented
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]

acceptance/bundle/resources/apps/output.txt

@@ -7,12 +7,12 @@ Deployment complete!
 
 >>> print_requests
 {
-  "method": "POST",
-  "path": "/api/2.0/apps",
   "body": {
     "description": "my_app_description",
     "name": "myapp"
-  }
+  },
+  "method": "POST",
+  "path": "/api/2.0/apps"
 }
 apps myapp name='myapp' description='my_app_description'
 
@@ -27,12 +27,11 @@ Deployment complete!
 
 >>> print_requests
 {
-  "method": "PATCH",
-  "path": "/api/2.0/apps/myapp",
   "body": {
     "description": "MY_APP_DESCRIPTION",
-    "name": "myapp",
-    "url": "myapp-123.cloud.databricksapps.com"
-  }
+    "name": "myapp"
+  },
+  "method": "PATCH",
+  "path": "/api/2.0/apps/myapp"
 }
 apps myapp name='myapp' description='MY_APP_DESCRIPTION'

acceptance/bundle/resources/apps/script

@@ -1,5 +1,6 @@
 print_requests() {
-    jq 'select(.method != "GET" and (.path | contains("/apps")))' < out.requests.txt
+    # url is output-only field that terraform adds but that is ignored by the backend
+    jq --sort-keys 'select(.method != "GET" and (.path | contains("/apps"))) | (.body.url = null | del(.body.url))' < out.requests.txt
     rm out.requests.txt
 }
 

acceptance/bundle/resources/pipelines/databricks.yml.tmpl

@@ -0,0 +1,10 @@
+bundle:
+  name: acc-$UNIQUE_NAME
+
+resources:
+  pipelines:
+    my:
+      name: test-pipeline-$UNIQUE_NAME
+      libraries:
+        - file:
+            path: "./foo.py"