Initial implementation of "direct" deployment backend

.github/workflows/push.yml

@@ -42,15 +42,24 @@ jobs:
           - macos-latest
           - ubuntu-latest
           - windows-latest
+        deployment:
+          - "terraform"
+          - "direct"
 
     steps:
       - name: Checkout repository and submodules
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+      - name: Create deployment-specific cache identifier
+        run: echo "${{ matrix.deployment }}" > deployment-type.txt
+
       - name: Setup Go
         uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
         with:
           go-version-file: go.mod
+          cache-dependency-path: |
+            go.sum
+            deployment-type.txt
 
       - name: Setup Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@@ -72,11 +81,15 @@ jobs:
         # and would like to run the tests as fast as possible. We run it on schedule as well, because that is what
         # populates the cache and cache may include test results.
         if: ${{ github.event_name == 'pull_request' || github.event_name == 'schedule' }}
+        env:
+          ENVFILTER: DATABRICKS_CLI_DEPLOYMENT=${{ matrix.deployment }}
         run: make test
 
       - name: Run tests with coverage
         # Still run 'make cover' on push to main and merge checks to make sure it does not get broken.
         if: ${{ github.event_name != 'pull_request' && github.event_name != 'schedule' }}
+        env:
+          ENVFILTER: DATABRICKS_CLI_DEPLOYMENT=${{ matrix.deployment }}
         run: make cover
 
       - name: Analyze slow tests

acceptance/bin/read_id.py

@@ -29,4 +29,19 @@ def print_resource_terraform(section, name):
             return
 
 
-print_resource_terraform(*sys.argv[1:])
+def print_resource_terranova(section, name):
+    filename = ".databricks/bundle/default/resources.json"
+    raw = open(filename).read()
+    data = json.loads(raw)
+    resources = data["resources"].get(section, {})
+    result = resources.get(name)
+    if result is None:
+        print(f"Resource {section=} {name=} not found. Available: {raw}")
+        return
+    print(result.get("__id__"))
+
+
+if os.environ.get("DATABRICKS_CLI_DEPLOYMENT", "").startswith("direct"):
+    print_resource_terranova(*sys.argv[1:])
+else:
+    print_resource_terraform(*sys.argv[1:])

acceptance/bin/read_state.py

@@ -13,17 +13,15 @@
 def print_resource_terraform(section, name, *attrs):
     resource_type = "databricks_" + section[:-1]
     filename = ".databricks/bundle/default/terraform/terraform.tfstate"
-    data = json.load(open(filename))
-    available = []
+    raw = open(filename).read()
+    data = json.loads(raw)
     found = 0
     for r in data["resources"]:
         r_type = r["type"]
         r_name = r["name"]
         if r_type != resource_type:
-            available.append((r_type, r_name))
             continue
         if r_name != name:
-            available.append((r_type, r_name))
             continue
         for inst in r["instances"]:
             attribute_values = inst.get("attributes")
@@ -35,4 +33,22 @@ def print_resource_terraform(section, name, *attrs):
         print(f"State not found for {section}.{name}")
 
 
-print_resource_terraform(*sys.argv[1:])
+def print_resource_terranova(section, name, *attrs):
+    filename = ".databricks/bundle/default/resources.json"
+    raw = open(filename).read()
+    data = json.loads(raw)
+    resources = data["resources"].get(section, {})
+    result = resources.get(name)
+    if result is None:
+        print(f"State not found for {section}.{name}")
+        return
+    state = result["state"]
+    state.setdefault("id", result.get("__id__"))
+    values = [f"{x}={state.get(x)!r}" for x in attrs]
+    print(section, name, " ".join(values))
+
+
+if os.environ.get("DATABRICKS_CLI_DEPLOYMENT", "").startswith("direct"):
+    print_resource_terranova(*sys.argv[1:])
+else:
+    print_resource_terraform(*sys.argv[1:])

acceptance/bundle/artifacts/whl_change_version/output.txt

@@ -41,10 +41,11 @@ dist/my_test_code-0.1.0-py3-none-any.whl
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/test.toml"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME"
 
 >>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0
 
@@ -88,7 +89,7 @@ dist/my_test_code-0.2.0-py3-none-any.whl
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME"
 
 === Restore config to target old wheel
 >>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl
@@ -135,4 +136,4 @@ dist/my_test_code-0.2.0-py3-none-any.whl
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME"

acceptance/bundle/artifacts/whl_change_version/test.toml

@@ -0,0 +1,3 @@
+[[Repls]]
+Old = '(resources.json|terraform.tfstate)'
+New = 'STATE_FILENAME'

acceptance/bundle/artifacts/whl_dynamic/test.toml

@@ -1,3 +1,5 @@
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # need to sort tasks by key like terraform does
+
 [[Repls]]
 Old = '\\\\'
 New = '/'

acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/test.toml

@@ -1,4 +1,5 @@
 BundleConfig.default_name = ""
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # need to sort tasks by key
 
 [[Repls]]
 Old = '\\'

acceptance/bundle/debug/direct/out.stderr.txt

@@ -0,0 +1,93 @@
+
+>>> [CLI] bundle validate --debug
+10:07:59 Info: start pid=12345 version=[DEV_VERSION] args="[CLI], bundle, validate, --debug"
+10:07:59 Debug: Found bundle root at [TEST_TMP_DIR] (file [TEST_TMP_DIR]/databricks.yml) pid=12345
+10:07:59 Info: Phase: load pid=12345
+10:07:59 Debug: Apply pid=12345 mutator=EntryPoint
+10:07:59 Debug: Apply pid=12345 mutator=scripts.preinit
+10:07:59 Debug: No script defined for preinit, skipping pid=12345 mutator=scripts.preinit
+10:07:59 Debug: Apply pid=12345 mutator=ProcessRootIncludes
+10:07:59 Debug: Apply pid=12345 mutator=VerifyCliVersion
+10:07:59 Debug: Apply pid=12345 mutator=EnvironmentsToTargets
+10:07:59 Debug: Apply pid=12345 mutator=ComputeIdToClusterId
+10:07:59 Debug: Apply pid=12345 mutator=InitializeVariables
+10:07:59 Debug: Apply pid=12345 mutator=DefineDefaultTarget(default)
+10:07:59 Debug: Apply pid=12345 mutator=validate:unique_resource_keys
+10:07:59 Debug: Apply pid=12345 mutator=SelectDefaultTarget
+10:07:59 Debug: Apply pid=12345 mutator=SelectDefaultTarget mutator=SelectTarget(default)
+10:07:59 Debug: Apply pid=12345 mutator=<func>
+10:07:59 Info: Phase: initialize pid=12345
+10:07:59 Debug: Apply pid=12345 mutator=validate:AllResourcesHaveValues
+10:07:59 Debug: Apply pid=12345 mutator=validate:interpolation_in_auth_config
+10:07:59 Debug: Apply pid=12345 mutator=RewriteSyncPaths
+10:07:59 Debug: Apply pid=12345 mutator=SyncDefaultPath
+10:07:59 Debug: Apply pid=12345 mutator=SyncInferRoot
+10:07:59 Debug: Apply pid=12345 mutator=PopulateCurrentUser
+10:07:59 Debug: GET /api/2.0/preview/scim/v2/Me
+< HTTP/1.1 200 OK
+< {
+<   "id": "[USERID]",
+<   "userName": "[USERNAME]"
+< } pid=12345 mutator=PopulateCurrentUser sdk=true
+10:07:59 Debug: Apply pid=12345 mutator=LoadGitDetails
+10:07:59 Debug: Apply pid=12345 mutator=ApplySourceLinkedDeploymentPreset
+10:07:59 Debug: Apply pid=12345 mutator=DefineDefaultWorkspaceRoot
+10:07:59 Debug: Apply pid=12345 mutator=ExpandWorkspaceRoot
+10:07:59 Debug: Apply pid=12345 mutator=DefaultWorkspacePaths
+10:07:59 Debug: Apply pid=12345 mutator=PrependWorkspacePrefix
+10:07:59 Debug: Apply pid=12345 mutator=RewriteWorkspacePrefix
+10:07:59 Debug: Apply pid=12345 mutator=SetVariables
+10:07:59 Debug: Apply pid=12345 mutator=ResolveVariableReferences
+10:07:59 Debug: Apply pid=12345 mutator=ResolveResourceReferences
+10:07:59 Debug: Apply pid=12345 mutator=ResolveVariableReferences
+10:07:59 Debug: Apply pid=12345 mutator=validate:volume-path
+10:07:59 Debug: Apply pid=12345 mutator=ApplyTargetMode
+10:07:59 Debug: Apply pid=12345 mutator=ConfigureWSFS
+10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources
+10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=ResolveVariableReferences(resources)
+10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=NormalizePaths
+10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=TranslatePathsDashboards
+10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load)
+10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(init)
+10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load_resources)
+10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(apply_mutators)
+10:07:59 Debug: Apply pid=12345 mutator=CheckPermissions
+10:07:59 Debug: Apply pid=12345 mutator=TranslatePaths
+10:07:59 Debug: Apply pid=12345 mutator=PythonWrapperWarning
+10:07:59 Debug: Apply pid=12345 mutator=ApplyArtifactsDynamicVersion
+10:07:59 Debug: Apply pid=12345 mutator=artifacts.Prepare
+10:07:59 Info: No local tasks in databricks.yml config, skipping auto detect pid=12345 mutator=artifacts.Prepare
+10:07:59 Debug: Apply pid=12345 mutator=apps.Validate
+10:07:59 Debug: Apply pid=12345 mutator=ValidateTargetMode
+10:07:59 Debug: Apply pid=12345 mutator=ValidateSharedRootPermissions
+10:07:59 Debug: Apply pid=12345 mutator=metadata.AnnotateJobs
+10:07:59 Debug: Apply pid=12345 mutator=metadata.AnnotatePipelines
+10:07:59 Debug: Apply pid=12345 mutator=scripts.postinit
+10:07:59 Debug: No script defined for postinit, skipping pid=12345 mutator=scripts.postinit
+10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly)
+10:07:59 Debug: ApplyParallel pid=12345 mutator=validate:files_to_sync
+10:07:59 Debug: ApplyParallel pid=12345 mutator=validate:folder_permissions
+10:07:59 Debug: ApplyParallel pid=12345 mutator=validate:validate_sync_patterns
+10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:job_cluster_key_defined
+10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:job_task_cluster_spec
+10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:artifact_paths
+10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files
+< HTTP/1.1 404 Not Found
+< {
+<   "message": "Workspace path not found"
+< } pid=12345 mutator=validate:files_to_sync sdk=true
+10:07:59 Debug: non-retriable error: Workspace path not found pid=12345 mutator=validate:files_to_sync sdk=true
+10:07:59 Debug: POST /api/2.0/workspace/mkdirs
+> {
+>   "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files"
+> }
+< HTTP/1.1 200 OK pid=12345 mutator=validate:files_to_sync sdk=true
+10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files
+< HTTP/1.1 200 OK
+< {
+<   "object_type": "DIRECTORY",
+<   "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files"
+< } pid=12345 mutator=validate:files_to_sync sdk=true
+10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files has type directory (ID: 0) pid=12345 mutator=validate:files_to_sync
+10:07:59 Info: completed execution pid=12345 exit_code=0
+10:07:59 Debug: no telemetry logs to upload pid=12345

acceptance/bundle/debug/direct/output.txt

@@ -0,0 +1,25 @@
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Validation OK!
+
+>>> diff.py [TESTROOT]/bundle/debug/direct/../tf/out.stderr.txt out.stderr.txt
+--- [TESTROOT]/bundle/debug/direct/../tf/out.stderr.txt
++++ out.stderr.txt
+@@ -1,2 +1,4 @@
++
++>>> [CLI] bundle validate --debug
+ 10:07:59 Info: start pid=12345 version=[DEV_VERSION] args="[CLI], bundle, validate, --debug"
+ 10:07:59 Debug: Found bundle root at [TEST_TMP_DIR] (file [TEST_TMP_DIR]/databricks.yml) pid=12345
+@@ -61,8 +63,4 @@
+ 10:07:59 Debug: Apply pid=12345 mutator=metadata.AnnotateJobs
+ 10:07:59 Debug: Apply pid=12345 mutator=metadata.AnnotatePipelines
+-10:07:59 Debug: Apply pid=12345 mutator=terraform.Initialize
+-10:07:59 Debug: Using Terraform from DATABRICKS_TF_EXEC_PATH at [TERRAFORM] pid=12345 mutator=terraform.Initialize
+-10:07:59 Debug: Using Terraform CLI config from DATABRICKS_TF_CLI_CONFIG_FILE at [DATABRICKS_TF_CLI_CONFIG_FILE] pid=12345 mutator=terraform.Initialize
+-10:07:59 Debug: Environment variables for Terraform: ...redacted... pid=12345 mutator=terraform.Initialize
+ 10:07:59 Debug: Apply pid=12345 mutator=scripts.postinit
+ 10:07:59 Debug: No script defined for postinit, skipping pid=12345 mutator=scripts.postinit

acceptance/bundle/debug/direct/script

@@ -0,0 +1,2 @@
+trace $CLI bundle validate --debug 2> out.stderr.txt
+trace diff.py $TESTDIR/../tf/out.stderr.txt out.stderr.txt

acceptance/bundle/debug/direct/test.toml

@@ -0,0 +1 @@
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["direct-exp"]

acceptance/bundle/debug/tf/test.toml

@@ -0,0 +1 @@
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]

acceptance/bundle/deploy/dashboard/test.toml

@@ -0,0 +1 @@
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # dashboard not supported yet

acceptance/bundle/deploy/fail-on-active-runs/test.toml

@@ -1,5 +1,8 @@
 RecordRequests = true
 
+# --fail-on-active-runs not implemented yet
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 [[Server]]
 Pattern = "GET /api/2.2/jobs/runs/list"
 Response.Body = '''

acceptance/bundle/deploy/jobs/check-metadata/test.toml

@@ -1,6 +1,8 @@
 Local = false
 Cloud = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # require "bundle summary"
+
 Ignore = [
     "databricks.yml",
     "a/b/resources.yml",

acceptance/bundle/deploy/jobs/double-underscore-keys/test.toml

@@ -1,6 +1,8 @@
 Local = true
 Cloud = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # needs investigation Error: deploying jobs.foo: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="Missing required field: settings.tasks.task_key."
+
 Ignore = [
     "databricks.yml",
 ]

acceptance/bundle/deploy/mlops-stacks/test.toml

@@ -3,6 +3,8 @@ Local=false
 
 Badness = "the newly initialized bundle from the 'mlops-stacks' template contains two validation warnings in the configuration"
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # requires "bundle summary"
+
 Ignore = [
     "config.json"
 ]

acceptance/bundle/deploy/pipeline/auto-approve/test.toml

@@ -1,6 +1,8 @@
 Local = true
 Cloud = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # requires "bundle summary"
+
 Ignore = [
     "databricks.yml"
 ]

acceptance/bundle/deploy/pipeline/recreate/test.toml

@@ -2,6 +2,8 @@ Local = true
 Cloud = true
 RequiresUnityCatalog = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 Ignore = [
     "databricks.yml"
 ]

acceptance/bundle/deploy/schema/auto-approve/test.toml

@@ -2,6 +2,8 @@ Local = true
 Cloud = true
 RequiresUnityCatalog = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # requires "bundle summary"
+
 Ignore = [
     "databricks.yml",
     "test-file-*.txt",

acceptance/bundle/deploy/secret-scope/test.toml

@@ -1,6 +1,8 @@
 Cloud = true
 Local = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
 Ignore = [
     "databricks.yml",
 ]

acceptance/bundle/deploy/volume/recreate/test.toml

@@ -2,6 +2,8 @@ Local = false
 Cloud = true
 RequiresUnityCatalog = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # volumes are not supported
+
 Ignore = [
     "databricks.yml",
 ]

acceptance/bundle/deployment/test.toml

@@ -1 +1,3 @@
 Cloud = true
+
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # summary,bind,unbind not implemented

acceptance/bundle/destroy/jobs-and-pipeline/test.toml

@@ -1,6 +1,8 @@
 Local = false
 Cloud = true
 
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # requires "bundle summary"
+
 Ignore = [
     "databricks.yml",
     "resources.yml",

acceptance/bundle/generate/dashboard-inplace/test.toml

@@ -1,3 +1,5 @@
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # dashboards not supported yet
+
 [[Repls]]
 Old = "[0-9a-f]{32}"
 New = "[DASHBOARD_ID]"

acceptance/bundle/includes/yml_outside_root/output.txt

@@ -44,8 +44,5 @@ Validation OK!
     "bundle_root_path": "."
   },
   "name": "yml_outside_root",
-  "target": "default",
-  "terraform": {
-    "exec_path": "[TERRAFORM]"
-  }
+  "target": "default"
 }

acceptance/bundle/includes/yml_outside_root/script

@@ -2,4 +2,4 @@ cd root
 trace $CLI bundle validate
 trace $CLI bundle validate -o json | jq '.resources.jobs[] | select(.name == "include_outside_root")'
 trace $CLI bundle validate -o json | jq '.sync'
-trace $CLI bundle validate -o json | jq '.bundle'
+trace $CLI bundle validate -o json | jq '.bundle' | jq 'del(.terraform)'

acceptance/bundle/python/restricted-execution/test.toml

@@ -0,0 +1,2 @@
+# "bundle summary" is not implemented
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]