diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 7565481c47..013a741375 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -15,5 +15,6 @@ * Fixed normalising requirements file path in dependencies section ([#2861](https://github.com/databricks/cli/pull/2861)) * Fix default-python template not to add environments when serverless=yes and include\_python=no ([#2866](https://github.com/databricks/cli/pull/2866)) * Fixed handling of Unicode characters in Python support ([#2873](https://github.com/databricks/cli/pull/2873)) +* Added support for secret scopes in DABs ([#2744](https://github.com/databricks/cli/pull/2744)) ### API Changes diff --git a/acceptance/bundle/deploy/secret-scope/backend-type/databricks.yml b/acceptance/bundle/deploy/secret-scope/backend-type/databricks.yml new file mode 100644 index 0000000000..a8804c64c1 --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/backend-type/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: deploy-secret-scope-azure-backend + +resources: + secret_scopes: + secret_scope_azure: + name: test-secrets-azure-backend + backend_type: "AZURE_KEYVAULT" + keyvault_metadata: + resource_id: my_azure_keyvault_id + dns_name: my_azure_keyvault_dns_name diff --git a/acceptance/bundle/deploy/secret-scope/backend-type/output.txt b/acceptance/bundle/deploy/secret-scope/backend-type/output.txt new file mode 100644 index 0000000000..f66a537ff1 --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/backend-type/output.txt @@ -0,0 +1,16 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-secret-scope-azure-backend/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq -s .[] | select(.path=="/api/2.0/secrets/scopes/create") | .body out.requests.txt +{ + "backend_azure_keyvault": { + "dns_name": "my_azure_keyvault_dns_name", + "resource_id": "my_azure_keyvault_id" + }, + "scope": "test-secrets-azure-backend", + "scope_backend_type": "AZURE_KEYVAULT" +} diff --git a/acceptance/bundle/deploy/secret-scope/backend-type/script b/acceptance/bundle/deploy/secret-scope/backend-type/script new file mode 100644 index 0000000000..ae11605f3c --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/backend-type/script @@ -0,0 +1,3 @@ +trace $CLI bundle deploy +trace jq -s '.[] | select(.path=="/api/2.0/secrets/scopes/create") | .body' out.requests.txt +rm out.requests.txt diff --git a/acceptance/bundle/deploy/secret-scope/backend-type/test.toml b/acceptance/bundle/deploy/secret-scope/backend-type/test.toml new file mode 100644 index 0000000000..34ad85ab0f --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/backend-type/test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false + +RecordRequests = true diff --git a/acceptance/bundle/deploy/secret-scope/databricks.yml.tmpl b/acceptance/bundle/deploy/secret-scope/databricks.yml.tmpl new file mode 100644 index 0000000000..34ce83b56b --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: deploy-secret-scope-test-$UNIQUE_NAME + +resources: + secret_scopes: + secret_scope1: + name: $SECRET_SCOPE_NAME + backend_type: "DATABRICKS" + permissions: + - user_name: admins + level: WRITE + - user_name: users + level: READ diff --git a/acceptance/bundle/deploy/secret-scope/output.txt b/acceptance/bundle/deploy/secret-scope/output.txt new file mode 100644 index 0000000000..9c38113bf4 --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/output.txt @@ -0,0 +1,53 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-secret-scope-test-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle summary --output json +{ + "backend_type": "DATABRICKS", + "modified_status": "created", + "name": "my-secrets-[UUID]", + "permissions": [ + { + "level": "WRITE", + "user_name": "admins" + }, + { + "level": "READ", + "user_name": "users" + } + ] +} + +>>> [CLI] secrets list-scopes -o json +{ + "backend_type": "DATABRICKS", + "name": "my-secrets-[UUID]" +} + +>>> [CLI] secrets list-acls my-secrets-[UUID] +{"permission":"MANAGE","principal":"[USERNAME]"} +{"permission":"READ","principal":"users"} +{"permission":"WRITE","principal":"admins"} + +>>> [CLI] secrets put-secret my-secrets-[UUID] my-key --string-value my-secret-value + +>>> [CLI] secrets get-secret my-secrets-[UUID] my-key +{ + "key":"my-key", + "value":"bXktc2VjcmV0LXZhbHVl" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete secret_acl secret_acl_secret_scope1_0 + delete secret_acl secret_acl_secret_scope1_1 + delete secret_scope secret_scope1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-secret-scope-test-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/secret-scope/permissions/databricks.yml.tmpl b/acceptance/bundle/deploy/secret-scope/permissions/databricks.yml.tmpl new file mode 100644 index 0000000000..5f6b4a8ec3 --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/permissions/databricks.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: deploy-secret-scope-with-permissions + +resources: + secret_scopes: + secret_scope_azure: + name: test-secrets-permissions + +permissions: + - user_name: $CURRENT_USER_NAME + level: CAN_MANAGE + - group_name: users + level: CAN_VIEW + - group_name: admins + level: CAN_MANAGE diff --git a/acceptance/bundle/deploy/secret-scope/permissions/output.txt b/acceptance/bundle/deploy/secret-scope/permissions/output.txt new file mode 100644 index 0000000000..f00594d235 --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/permissions/output.txt @@ -0,0 +1,10 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-secret-scope-with-permissions/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq -s -c .[] | select(.path=="/api/2.0/secrets/acls/put") | .body out.requests.txt +{"permission":"MANAGE","principal":"admins","scope":"test-secrets-permissions"} +{"permission":"READ","principal":"users","scope":"test-secrets-permissions"} diff --git a/acceptance/bundle/deploy/secret-scope/permissions/script b/acceptance/bundle/deploy/secret-scope/permissions/script new file mode 100644 index 0000000000..b6e6e9d881 --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/permissions/script @@ -0,0 +1,4 @@ +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy #--log-level TRACE +trace jq -s -c '.[] | select(.path=="/api/2.0/secrets/acls/put") | .body' out.requests.txt | sort +rm out.requests.txt diff --git a/acceptance/bundle/deploy/secret-scope/permissions/test.toml b/acceptance/bundle/deploy/secret-scope/permissions/test.toml new file mode 100644 index 0000000000..aedaae4045 --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/permissions/test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false +RecordRequests = true + +[[Server]] +Pattern = "PUT /api/2.0/permissions/directories/{objectId}" diff --git a/acceptance/bundle/deploy/secret-scope/script b/acceptance/bundle/deploy/secret-scope/script new file mode 100644 index 0000000000..9a66467832 --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/script @@ -0,0 +1,21 @@ +SECRET_SCOPE_NAME="my-secrets-$(uuid)" +if [ -z "$CLOUD_ENV" ]; then + SECRET_SCOPE_NAME="my-secrets-6260d50f-e8ff-4905-8f28-812345678903" # use hard-coded uuid when running locally +fi +export SECRET_SCOPE_NAME + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +trace $CLI bundle summary --output json | jq '.resources.secret_scopes.secret_scope1' +trace $CLI secrets list-scopes -o json | jq --arg value ${SECRET_SCOPE_NAME} '.[] | select(.name == $value)' + +trace $CLI secrets list-acls ${SECRET_SCOPE_NAME} | jq -c '.[]' | sort + +trace $CLI secrets put-secret ${SECRET_SCOPE_NAME} my-key --string-value "my-secret-value" +trace $CLI secrets get-secret ${SECRET_SCOPE_NAME} my-key diff --git a/acceptance/bundle/deploy/secret-scope/test.toml b/acceptance/bundle/deploy/secret-scope/test.toml new file mode 100644 index 0000000000..4d7598a227 --- /dev/null +++ b/acceptance/bundle/deploy/secret-scope/test.toml @@ -0,0 +1,75 @@ +Cloud = true +Local = true + +Ignore = [ + "databricks.yml", +] + +[[Server]] +Pattern = "POST /api/2.0/secrets/scopes/create" + +[[Server]] +Pattern = "GET /api/2.0/secrets/scopes/list" +Response.Body = ''' +{ + "scopes": [ + { + "backend_type": "DATABRICKS", + "name": "my-secrets-6260d50f-e8ff-4905-8f28-812345678903" + }, + { + "backend_type": "AZURE_KEYVAULT", + "name": "test-secrets-azure-backend" + }, + { + "backend_type": "DATABRICKS", + "name": "test-secrets-permissions" + } + ] +} +''' + +[[Server]] +Pattern = "POST /api/2.0/secrets/scopes/delete" + +[[Server]] +Pattern = "POST /api/2.0/secrets/put" + +[[Server]] +Pattern = "POST /api/2.0/secrets/acls/put" + +[[Server]] +Pattern = "GET /api/2.0/secrets/get" +Response.Body = ''' +{ + "key":"my-key", + "value":"bXktc2VjcmV0LXZhbHVl" +} +''' + +[[Server]] +Pattern = "GET /api/2.0/secrets/acls/list" +Response.Body = ''' +{ + "items": [ + { + "permission": "READ", + "principal": "users" + }, + { + "permission": "WRITE", + "principal": "admins" + }, + { + "permission": "MANAGE", + "principal": "[USERNAME]" + } + ] +} +''' + +[[Server]] +Pattern = "GET /api/2.0/secrets/acls/get" + +[[Server]] +Pattern = "POST /api/2.0/secrets/acls/delete" diff --git a/acceptance/bundle/deployment/bind/secret-scope/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/secret-scope/databricks.yml.tmpl new file mode 100644 index 0000000000..b615c2c5f6 --- /dev/null +++ b/acceptance/bundle/deployment/bind/secret-scope/databricks.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: bind-secret-scope-test-$UNIQUE_NAME + +resources: + secret_scopes: + secret_scope1: + name: $SECRET_SCOPE_NAME diff --git a/acceptance/bundle/deployment/bind/secret-scope/output.txt b/acceptance/bundle/deployment/bind/secret-scope/output.txt new file mode 100644 index 0000000000..53f66c3c8c --- /dev/null +++ b/acceptance/bundle/deployment/bind/secret-scope/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] secrets create-scope test-secret-scope-[UUID] + +>>> [CLI] bundle deployment bind secret_scope1 test-secret-scope-[UUID] --auto-approve +Updating deployment state... +Successfully bound secret_scope with an id 'test-secret-scope-[UUID]'. Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/bind-secret-scope-test-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] secrets list-scopes -o json +{ + "backend_type": "DATABRICKS", + "name": "test-secret-scope-[UUID]" +} + +>>> [CLI] bundle deployment unbind secret_scope1 +Updating deployment state... + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/bind-secret-scope-test-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +>>> [CLI] secrets list-scopes -o json +{ + "backend_type": "DATABRICKS", + "name": "test-secret-scope-[UUID]" +} + +>>> [CLI] secrets delete-scope test-secret-scope-[UUID] diff --git a/acceptance/bundle/deployment/bind/secret-scope/script b/acceptance/bundle/deployment/bind/secret-scope/script new file mode 100644 index 0000000000..1dd2233851 --- /dev/null +++ b/acceptance/bundle/deployment/bind/secret-scope/script @@ -0,0 +1,26 @@ +SECRET_SCOPE_NAME="test-secret-scope-$(uuid)" +if [ -z "$CLOUD_ENV" ]; then + SECRET_SCOPE_NAME="test-secret-scope-6260d50f-e8ff-4905-8f28-812345678903" # use hard-coded uuid when running locally +fi +export SECRET_SCOPE_NAME +envsubst < databricks.yml.tmpl > databricks.yml + +# Create a pre-defined volume: +trace $CLI secrets create-scope "${SECRET_SCOPE_NAME}" + +cleanup() { + trace $CLI secrets delete-scope "${SECRET_SCOPE_NAME}" +} +trap cleanup EXIT + +trace $CLI bundle deployment bind secret_scope1 "${SECRET_SCOPE_NAME}" --auto-approve + +trace $CLI bundle deploy + +trace $CLI secrets list-scopes -o json | jq --arg value ${SECRET_SCOPE_NAME} '.[] | select(.name == $value)' + +trace $CLI bundle deployment unbind secret_scope1 + +trace $CLI bundle destroy --auto-approve + +trace $CLI secrets list-scopes -o json | jq --arg value ${SECRET_SCOPE_NAME} '.[] | select(.name == $value)' diff --git a/acceptance/bundle/deployment/bind/secret-scope/test.toml b/acceptance/bundle/deployment/bind/secret-scope/test.toml new file mode 100644 index 0000000000..3c38ba78fc --- /dev/null +++ b/acceptance/bundle/deployment/bind/secret-scope/test.toml @@ -0,0 +1,26 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +Ignore = [ + "databricks.yml", +] + +[[Server]] +Pattern = "POST /api/2.0/secrets/scopes/create" + +[[Server]] +Pattern = "GET /api/2.0/secrets/scopes/list" +Response.Body = ''' +{ + "scopes": [ + { + "backend_type": "DATABRICKS", + "name": "test-secret-scope-6260d50f-e8ff-4905-8f28-812345678903" + } + ] +} +''' + +[[Server]] +Pattern = "POST /api/2.0/secrets/scopes/delete" diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index 3c438ce7b7..a015fdbfe1 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -51,6 +51,10 @@ var ( permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_USE", }, + "secret_scopes": { + permissions.CAN_MANAGE: "MANAGE", + permissions.CAN_VIEW: "READ", + }, } ) diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 6f32300631..575b9fb6a0 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -152,6 +152,11 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + SecretScopes: map[string]*resources.SecretScope{ + "secretScope1": { + Name: "secretScope1", + }, + }, }, }, SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"), @@ -318,8 +323,8 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) { nameField := resource.Elem().FieldByName("Name") resourceType := resources.Type().Field(i).Name - // Skip apps, as they are not renamed - if resourceType == "Apps" { + // Skip resources that are not renamed + if resourceType == "Apps" || resourceType == "SecretScopes" { continue } diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 9ff5d5faec..0349959dc7 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -43,6 +43,7 @@ func allResourceTypes(t *testing.T) []string { "quality_monitors", "registered_models", "schemas", + "secret_scopes", "volumes", }, resourceTypes, @@ -142,6 +143,7 @@ var allowList = []string{ "registered_models", "experiments", "schemas", + "secret_scopes", "volumes", } diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 204ca93127..a97fe6584d 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -24,6 +24,7 @@ type Resources struct { Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"` Apps map[string]*resources.App `json:"apps,omitempty"` + SecretScopes map[string]*resources.SecretScope `json:"secret_scopes,omitempty"` } type ConfigResource interface { @@ -86,6 +87,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["dashboards"], r.Dashboards), collectResourceMap(descriptions["volumes"], r.Volumes), collectResourceMap(descriptions["apps"], r.Apps), + collectResourceMap(descriptions["secret_scopes"], r.SecretScopes), } } @@ -157,6 +159,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) } } + for k := range r.SecretScopes { + if k == key { + found = append(found, r.SecretScopes[k]) + } + } + if len(found) == 0 { return nil, fmt.Errorf("no such resource: %s", key) } @@ -187,5 +195,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "dashboards": (&resources.Dashboard{}).ResourceDescription(), "volumes": (&resources.Volume{}).ResourceDescription(), "apps": (&resources.App{}).ResourceDescription(), + "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), } } diff --git a/bundle/config/resources/secret_scope.go b/bundle/config/resources/secret_scope.go new file mode 100644 index 0000000000..d9922a1f97 --- /dev/null +++ b/bundle/config/resources/secret_scope.go @@ -0,0 +1,100 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +type SecretScopePermissionLevel string + +// SecretScopePermission holds the permission level setting for a single principal. +// Multiple of these can be defined on any secret scope. +// Secret scopes permissions are mapped to Secret ACLs +type SecretScopePermission struct { + Level SecretScopePermissionLevel `json:"level"` + + UserName string `json:"user_name,omitempty"` + ServicePrincipalName string `json:"service_principal_name,omitempty"` + GroupName string `json:"group_name,omitempty"` +} + +type SecretScope struct { + // A unique name to identify the secret scope. + Name string `json:"name"` + + Permissions []SecretScopePermission `json:"permissions,omitempty"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + + // Secret scope configuration is explicitly defined here with individual fields + // to maintain API stability and prevent unintended configuration changes. + // This approach decouples our configuration from potential upstream model/SDK changes + // to `workspace.SecretScope`. While the upstream type serves as a response payload + // for workspace.ListScopesResponse, we adopt its field naming conventions + // for better developer experience compared to `workspace.CreateScope`. + + // The type of secret scope backend. + BackendType workspace.ScopeBackendType `json:"backend_type,omitempty"` + // The metadata for the secret scope if the type is `AZURE_KEYVAULT` + KeyvaultMetadata *workspace.AzureKeyVaultSecretScopeMetadata `json:"keyvault_metadata,omitempty"` +} + +func (s *SecretScope) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s SecretScope) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s SecretScope) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + // NOTE: Scope lookup by name is not directly supported by the Secret scopes API + // As of May 2025 there is no direct API method to retrieve a scope using its name as an identifier. + // While scope names serve as unique identifiers, the API only provides: + // - List operations that returns a list of scopes + // - Other operational methods (e.g., reading a secret from a scope and parsing error messages) + // + // The indirect methods are not semantically ideal for simple existence checks, so we use the list API here + scopes, err := w.Secrets.ListScopesAll(ctx) + if err != nil { + return false, nil + } + + for _, scope := range scopes { + if scope.Name == name { + return true, nil + } + } + + return false, nil +} + +func (s SecretScope) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "secret_scope", + PluralName: "secret_scopes", + SingularTitle: "Secret Scope", + PluralTitle: "Secret Scopes", + TerraformResourceName: "databricks_secret_scope", + } +} + +func (s SecretScope) TerraformResourceName() string { + return "databricks_secret_scope" +} + +func (s SecretScope) GetName() string { + return s.Name +} + +func (s SecretScope) GetURL() string { + // Secret scopes do not have a URL + return "" +} + +func (s SecretScope) InitializeURL(_ url.URL) { + // Secret scopes do not have a URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 92bd32e5c1..656cb6a680 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/cli/bundle/config/resources" @@ -168,6 +170,11 @@ func TestResourcesBindSupport(t *testing.T) { CreateServingEndpoint: serving.CreateServingEndpoint{}, }, }, + SecretScopes: map[string]*resources.SecretScope{ + "my_secret_scope": { + Name: "0", + }, + }, } unbindableResources := map[string]bool{"model": true} @@ -184,6 +191,9 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockAppsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockQualityMonitorsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockServingEndpointsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockSecretsAPI().EXPECT().ListScopesAll(mock.Anything).Return([]workspace.SecretScope{ + {Name: "0"}, + }, nil) allResources := supportedResources.AllResources() for _, group := range allResources { diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index a5821a240d..befee0b9d9 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -210,8 +210,19 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { } cur.Name = instance.Attributes.Name config.Resources.Apps[resource.Name] = cur + case "databricks_secret_scope": + if config.Resources.SecretScopes == nil { + config.Resources.SecretScopes = make(map[string]*resources.SecretScope) + } + cur := config.Resources.SecretScopes[resource.Name] + if cur == nil { + cur = &resources.SecretScope{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.Name = instance.Attributes.Name + config.Resources.SecretScopes[resource.Name] = cur case "databricks_permissions": case "databricks_grants": + case "databricks_secret_acl": // Ignore; no need to pull these back into the configuration. default: return fmt.Errorf("missing mapping for %s", resource.Type) @@ -279,6 +290,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { src.ModifiedStatus = resources.ModifiedStatusCreated } } + for _, src := range config.Resources.SecretScopes { + if src.ModifiedStatus == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } return nil } diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index bb7f149358..6467ce69e8 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -703,6 +703,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { {Attributes: stateInstanceAttributes{Name: "app1"}}, }, }, + { + Type: "databricks_secret_scope", + Mode: "managed", + Name: "test_secret_scope", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{Name: "secret_scope1"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -834,6 +842,11 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + SecretScopes: map[string]*resources.SecretScope{ + "test_secret_scope": { + Name: "test_secret_scope", + }, + }, }, } tfState := resourcesState{ @@ -1028,6 +1041,14 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, }, }, + SecretScopes: map[string]*resources.SecretScope{ + "test_secret_scope": { + Name: "test_secret_scope", + }, + "test_secret_scope_new": { + Name: "test_secret_scope_new", + }, + }, }, } tfState := resourcesState{ diff --git a/bundle/deploy/terraform/tfdyn/convert_secret_scope.go b/bundle/deploy/terraform/tfdyn/convert_secret_scope.go new file mode 100644 index 0000000000..97ebce7bba --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_secret_scope.go @@ -0,0 +1,71 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +type resourceSecretAcl struct { + schema.ResourceSecretAcl + DependsOn []string `json:"depends_on,omitempty"` +} + +type secretScopeConverter struct{} + +func convertPermissionsSecretScope(key, scopeName string, permissions []dyn.Value, out *schema.Resources) { + for idx, permission := range permissions { + level, _ := permission.Get("level").AsString() + userName, _ := permission.Get("user_name").AsString() + groupName, _ := permission.Get("group_name").AsString() + servicePrincipalName, _ := permission.Get("service_principal_name").AsString() + + principal := "" + if userName != "" { + principal = userName + } else if groupName != "" { + principal = groupName + } else if servicePrincipalName != "" { + principal = servicePrincipalName + } + + acl := &resourceSecretAcl{ + ResourceSecretAcl: schema.ResourceSecretAcl{ + Permission: level, + Principal: principal, + Scope: scopeName, + }, + DependsOn: []string{"databricks_secret_scope." + key}, + } + + aclKey := fmt.Sprintf("secret_acl_%s_%d", key, idx) + out.SecretAcl[aclKey] = acl + } +} + +func (s secretScopeConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(workspace.SecretScope{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "secret scope normalization diagnostic: %s", diag.Summary) + } + out.SecretScope[key] = vout.AsAny() + + // Configure permissions for this resource + scopeName, _ := vin.Get("name").AsString() + permissions, ok := vin.Get("permissions").AsSequence() + if ok && len(permissions) > 0 { + convertPermissionsSecretScope(key, scopeName, permissions, out) + } + + return nil +} + +func init() { + registerConverter("secret_scopes", secretScopeConverter{}) +} diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index c51c5c190a..052ad904e3 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,7 +1,7 @@ --- description: 'Configuration reference for databricks.yml' last_update: - date: 2025-04-01 + date: 2025-05-12 --- @@ -438,6 +438,10 @@ resources: - Map - The schema definitions for the bundle, where each key is the name of the schema. See [\_](/dev-tools/bundles/resources.md#schemas). +- - `secret_scopes` + - Map + - See [\_](#resourcessecret_scopes). + - - `volumes` - Map - The volume definitions for the bundle, where each key is the name of the volume. See [\_](/dev-tools/bundles/resources.md#volumes). @@ -445,6 +449,77 @@ resources: ::: +### resources.secret_scopes + +**`Type: Map`** + + + +```yaml +secret_scopes: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `backend_type` + - String + - The backend type the scope will be created with. If not specified, will default to `DATABRICKS` + +- - `keyvault_metadata` + - Map + - The metadata for the secret scope if the `backend_type` is `AZURE_KEYVAULT`. See [\_](#resourcessecret_scopesnamekeyvault_metadata). + +- - `name` + - String + - Scope name requested by the user. Scope names are unique. + +- - `permissions` + - Sequence + - The permissions to apply to the secret scope. Permissions are managed via secret scope ACLs. See [\_](#resourcessecret_scopesnamepermissions). + +::: + + +### resources.secret_scopes._name_.permissions + +**`Type: Sequence`** + +The permissions to apply to the secret scope. Permissions are managed via secret scope ACLs. + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. This field translates to a `principal` field in secret scope ACL. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The application ID of an active service principal. This field translates to a `principal` field in secret scope ACL. + +- - `user_name` + - String + - The name of the user that has the permission set in level. This field translates to a `principal` field in secret scope ACL. + +::: + + ## run_as **`Type: Map`** @@ -921,6 +996,10 @@ The resource definitions for the target. - Map - The schema definitions for the bundle, where each key is the name of the schema. See [\_](/dev-tools/bundles/resources.md#schemas). +- - `secret_scopes` + - Map + - See [\_](#targetsnameresourcessecret_scopes). + - - `volumes` - Map - The volume definitions for the bundle, where each key is the name of the volume. See [\_](/dev-tools/bundles/resources.md#volumes). @@ -928,6 +1007,77 @@ The resource definitions for the target. ::: +### targets._name_.resources.secret_scopes + +**`Type: Map`** + + + +```yaml +secret_scopes: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `backend_type` + - String + - The backend type the scope will be created with. If not specified, will default to `DATABRICKS` + +- - `keyvault_metadata` + - Map + - The metadata for the secret scope if the `backend_type` is `AZURE_KEYVAULT`. See [\_](#targetsnameresourcessecret_scopesnamekeyvault_metadata). + +- - `name` + - String + - Scope name requested by the user. Scope names are unique. + +- - `permissions` + - Sequence + - The permissions to apply to the secret scope. Permissions are managed via secret scope ACLs. See [\_](#targetsnameresourcessecret_scopesnamepermissions). + +::: + + +### targets._name_.resources.secret_scopes._name_.permissions + +**`Type: Sequence`** + +The permissions to apply to the secret scope. Permissions are managed via secret scope ACLs. + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. This field translates to a `principal` field in secret scope ACL. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The application ID of an active service principal. This field translates to a `principal` field in secret scope ACL. + +- - `user_name` + - String + - The name of the user that has the permission set in level. This field translates to a `principal` field in secret scope ACL. + +::: + + ### targets._name_.run_as **`Type: Map`** diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index b8f7ec5dd8..76df1a1783 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,7 +1,7 @@ --- description: 'Learn about resources supported by Databricks Asset Bundles and how to configure them.' last_update: - date: 2025-04-01 + date: 2025-05-12 --- @@ -508,19 +508,19 @@ for deployment to the app compute. - - `group_name` - String - - The name of the group that has the permission set in level. + - - - `level` - String - - The allowed permission for user, group, service principal defined for this permission. + - - - `service_principal_name` - String - - The name of the service principal that has the permission set in level. + - - - `user_name` - String - - The name of the user that has the permission set in level. + - ::: @@ -1195,7 +1195,7 @@ If not specified at cluster creation, a set of default values will be used. - - `use_preemptible_executors` - Boolean - - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the 'availability' field instead. + - This field is deprecated - - `zone_id` - String @@ -1226,7 +1226,7 @@ If `cluster_log_conf` is specified, init script logs are sent to `/ - - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`. See [\_](#clustersnameinit_scriptsdbfs). + - This field is deprecated - - `file` - Map @@ -1272,28 +1272,6 @@ Contains the Azure Data Lake Storage destination path ::: -### clusters._name_.init_scripts.dbfs - -**`Type: Map`** - -destination needs to be provided. e.g. -`{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }` - - - -:::list-table - -- - Key - - Type - - Description - -- - `destination` - - String - - dbfs destination, e.g. `dbfs:/my/path` - -::: - - ### clusters._name_.init_scripts.file **`Type: Map`** @@ -1446,19 +1424,19 @@ destination needs to be provided, e.g. - - `group_name` - String - - The name of the group that has the permission set in level. + - - - `level` - String - - The allowed permission for user, group, service principal defined for this permission. + - - - `service_principal_name` - String - - The name of the service principal that has the permission set in level. + - - - `user_name` - String - - The name of the user that has the permission set in level. + - ::: @@ -1615,19 +1593,19 @@ In addition, if you attempt to deploy a bundle that contains a dashboard JSON fi - - `group_name` - String - - The name of the group that has the permission set in level. + - - - `level` - String - - The allowed permission for user, group, service principal defined for this permission. + - - - `service_principal_name` - String - - The name of the service principal that has the permission set in level. + - - - `user_name` - String - - The name of the user that has the permission set in level. + - ::: @@ -1717,19 +1695,19 @@ resources: - - `group_name` - String - - The name of the group that has the permission set in level. + - - - `level` - String - - The allowed permission for user, group, service principal defined for this permission. + - - - `service_principal_name` - String - - The name of the service principal that has the permission set in level. + - - - `user_name` - String - - The name of the user that has the permission set in level. + - ::: @@ -1808,7 +1786,7 @@ jobs: - - `format` - String - - Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `"MULTI_TASK"`. + - This field is deprecated - - `git_source` - Map @@ -1820,7 +1798,7 @@ jobs: - - `job_clusters` - Sequence - - A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. If more than 100 job clusters are available, you can paginate through them using :method:jobs/get. See [\_](#jobsnamejob_clusters). + - A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. See [\_](#jobsnamejob_clusters). - - `max_concurrent_runs` - Integer @@ -1840,7 +1818,7 @@ jobs: - - `performance_target` - String - - PerformanceTarget defines how performant or cost efficient the execution of run on serverless should be. + - The performance mode on a serverless job. This field determines the level of compute performance or cost-efficiency for the run. * `STANDARD`: Enables cost-efficient execution of serverless workloads. * `PERFORMANCE_OPTIMIZED`: Prioritizes fast startup and execution times through rapid scaling and optimized cluster performance. - - `permissions` - Sequence @@ -1864,7 +1842,7 @@ jobs: - - `tasks` - Sequence - - A list of task specifications to be executed by this job. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. See [\_](#jobsnametasks). + - A list of task specifications to be executed by this job. It supports up to 1000 elements in write endpoints (:method:jobs/create, :method:jobs/reset, :method:jobs/update, :method:jobs/submit). Read endpoints return only 100 tasks. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. See [\_](#jobsnametasks). - - `timeout_seconds` - Integer @@ -1960,7 +1938,7 @@ An optional set of email addresses that is notified when runs of this job begin - - `no_alert_for_skipped_runs` - Boolean - - If true, do not send email to recipients specified in `on_failure` if the run is skipped. This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. + - This field is deprecated - - `on_duration_warning_threshold_exceeded` - Sequence @@ -2008,7 +1986,7 @@ For other serverless tasks, the task environment is required to be specified usi - - `spec` - Map - - The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. In this minimal environment spec, only pip dependencies are supported. See [\_](#jobsnameenvironmentsspec). + - The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines. (Note: DLT uses a copied version of the Environment proto below, at //spark/pipelines/api/protos/copied/libraries-environments-copy.proto) In this minimal environment spec, only pip dependencies are supported. See [\_](#jobsnameenvironmentsspec). ::: @@ -2017,7 +1995,8 @@ For other serverless tasks, the task environment is required to be specified usi **`Type: Map`** -The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. +The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines. +(Note: DLT uses a copied version of the Environment proto below, at //spark/pipelines/api/protos/copied/libraries-environments-copy.proto) In this minimal environment spec, only pip dependencies are supported. @@ -2036,6 +2015,10 @@ In this minimal environment spec, only pip dependencies are supported. - Sequence - List of pip dependencies, as supported by the version of pip in this environment. +- - `jar_dependencies` + - Sequence + - List of jar dependencies, should be string representing volume paths. For example: `/Volumes/path/to/test.jar`. + ::: @@ -2083,7 +2066,7 @@ Note: dbt and SQL File tasks support only version-controlled sources. If dbt or - - `job_source` - Map - - The source of the job specification in the remote repository when the job is source controlled. See [\_](#jobsnamegit_sourcejob_source). + - This field is deprecated ::: @@ -2109,35 +2092,6 @@ Read-only state of the remote repository at the time the job was run. This field ::: -### jobs._name_.git_source.job_source - -**`Type: Map`** - -The source of the job specification in the remote repository when the job is source controlled. - - - -:::list-table - -- - Key - - Type - - Description - -- - `dirty_state` - - String - - Dirty state indicates the job is not fully synced with the job specification in the remote repository. Possible values are: * `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced. * `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced. - -- - `import_from_git_branch` - - String - - Name of the branch which the job is imported from. - -- - `job_config_path` - - String - - Path of the job YAML file that contains the job specification. - -::: - - ### jobs._name_.health **`Type: Map`** @@ -2193,7 +2147,6 @@ An optional set of health rules that can be defined for this job. **`Type: Sequence`** A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. -If more than 100 job clusters are available, you can paginate through them using :method:jobs/get. @@ -2702,7 +2655,7 @@ If not specified at cluster creation, a set of default values will be used. - - `use_preemptible_executors` - Boolean - - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the 'availability' field instead. + - This field is deprecated - - `zone_id` - String @@ -2733,7 +2686,7 @@ If `cluster_log_conf` is specified, init script logs are sent to `/ - - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`. See [\_](#jobsnamejob_clustersnew_clusterinit_scriptsdbfs). + - This field is deprecated - - `file` - Map @@ -2779,28 +2732,6 @@ Contains the Azure Data Lake Storage destination path ::: -### jobs._name_.job_clusters.new_cluster.init_scripts.dbfs - -**`Type: Map`** - -destination needs to be provided. e.g. -`{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }` - - - -:::list-table - -- - Key - - Type - - Description - -- - `destination` - - String - - dbfs destination, e.g. `dbfs:/my/path` - -::: - - ### jobs._name_.job_clusters.new_cluster.init_scripts.file **`Type: Map`** @@ -3049,19 +2980,19 @@ Job-level parameter definitions - - `group_name` - String - - The name of the group that has the permission set in level. + - - - `level` - String - - The allowed permission for user, group, service principal defined for this permission. + - - - `service_principal_name` - String - - The name of the service principal that has the permission set in level. + - - - `user_name` - String - - The name of the user that has the permission set in level. + - ::: @@ -3148,7 +3079,8 @@ An optional periodic schedule for this job. The default behavior is that the job **`Type: Sequence`** A list of task specifications to be executed by this job. -If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. +It supports up to 1000 elements in write endpoints (:method:jobs/create, :method:jobs/reset, :method:jobs/update, :method:jobs/submit). +Read endpoints return only 100 tasks. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. @@ -3166,6 +3098,10 @@ If more than 100 tasks are available, you can paginate through them using :metho - Map - The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and does not support retries or notifications. See [\_](#jobsnametaskscondition_task). +- - `dashboard_task` + - Map + - The task refreshes a dashboard and sends a snapshot to subscribers. See [\_](#jobsnametasksdashboard_task). + - - `dbt_task` - Map - The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. See [\_](#jobsnametasksdbt_task). @@ -3200,7 +3136,7 @@ If more than 100 tasks are available, you can paginate through them using :metho - - `gen_ai_compute_task` - Map - - Next field: 9. See [\_](#jobsnametasksgen_ai_compute_task). + - See [\_](#jobsnametasksgen_ai_compute_task). - - `health` - Map @@ -3238,6 +3174,10 @@ If more than 100 tasks are available, you can paginate through them using :metho - Map - The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported. See [\_](#jobsnametaskspipeline_task). +- - `power_bi_task` + - Map + - The task triggers a Power BI semantic model update when the `power_bi_task` field is present. See [\_](#jobsnametaskspower_bi_task). + - - `python_wheel_task` - Map - The task runs a Python wheel when the `python_wheel_task` field is present. See [\_](#jobsnametaskspython_wheel_task). @@ -3349,6 +3289,89 @@ The condition task does not require a cluster to execute and does not support re ::: +### jobs._name_.tasks.dashboard_task + +**`Type: Map`** + +The task refreshes a dashboard and sends a snapshot to subscribers. + + + +:::list-table + +- - Key + - Type + - Description + +- - `dashboard_id` + - String + - + +- - `subscription` + - Map + - See [\_](#jobsnametasksdashboard_tasksubscription). + +- - `warehouse_id` + - String + - Optional: The warehouse id to execute the dashboard with for the schedule. If not specified, the default warehouse of the dashboard will be used. + +::: + + +### jobs._name_.tasks.dashboard_task.subscription + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `custom_subject` + - String + - Optional: Allows users to specify a custom subject line on the email sent to subscribers. + +- - `paused` + - Boolean + - When true, the subscription will not send emails. + +- - `subscribers` + - Sequence + - See [\_](#jobsnametasksdashboard_tasksubscriptionsubscribers). + +::: + + +### jobs._name_.tasks.dashboard_task.subscription.subscribers + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `destination_id` + - String + - + +- - `user_name` + - String + - + +::: + + ### jobs._name_.tasks.dbt_task **`Type: Map`** @@ -3436,7 +3459,7 @@ An optional set of email addresses that is notified when runs of this task begin - - `no_alert_for_skipped_runs` - Boolean - - If true, do not send email to recipients specified in `on_failure` if the run is skipped. This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. + - This field is deprecated - - `on_duration_warning_threshold_exceeded` - Sequence @@ -3494,7 +3517,7 @@ The task executes a nested task for every input provided when the `for_each_task **`Type: Map`** -Next field: 9 + @@ -3510,7 +3533,7 @@ Next field: 9 - - `compute` - Map - - Next field: 4. See [\_](#jobsnametasksgen_ai_compute_taskcompute). + - See [\_](#jobsnametasksgen_ai_compute_taskcompute). - - `dl_runtime_image` - String @@ -3543,7 +3566,7 @@ Next field: 9 **`Type: Map`** -Next field: 4 + @@ -3639,7 +3662,7 @@ The default value is an empty list. - - `egg` - String - - Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above. + - This field is deprecated - - `jar` - String @@ -4233,7 +4256,7 @@ If not specified at cluster creation, a set of default values will be used. - - `use_preemptible_executors` - Boolean - - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the 'availability' field instead. + - This field is deprecated - - `zone_id` - String @@ -4264,7 +4287,7 @@ If `cluster_log_conf` is specified, init script logs are sent to `/ - - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`. See [\_](#jobsnametasksnew_clusterinit_scriptsdbfs). + - This field is deprecated - - `file` - Map @@ -4310,28 +4333,6 @@ Contains the Azure Data Lake Storage destination path ::: -### jobs._name_.tasks.new_cluster.init_scripts.dbfs - -**`Type: Map`** - -destination needs to be provided. e.g. -`{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }` - - - -:::list-table - -- - Key - - Type - - Description - -- - `destination` - - String - - dbfs destination, e.g. `dbfs:/my/path` - -::: - - ### jobs._name_.tasks.new_cluster.init_scripts.file **`Type: Map`** @@ -4601,11 +4602,11 @@ The task triggers a pipeline update when the `pipeline_task` field is present. O ::: -### jobs._name_.tasks.python_wheel_task +### jobs._name_.tasks.power_bi_task **`Type: Map`** -The task runs a Python wheel when the `python_wheel_task` field is present. +The task triggers a Power BI semantic model update when the `power_bi_task` field is present. @@ -4615,30 +4616,34 @@ The task runs a Python wheel when the `python_wheel_task` field is present. - Type - Description -- - `entry_point` +- - `connection_resource_name` - String - - Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()` + - The resource name of the UC connection to authenticate from Databricks to Power BI -- - `named_parameters` +- - `power_bi_model` - Map - - Command-line parameters passed to Python wheel task in the form of `["--name=task", "--data=dbfs:/path/to/data.json"]`. Leave it empty if `parameters` is not null. + - The semantic model to update. See [\_](#jobsnametaskspower_bi_taskpower_bi_model). -- - `package_name` - - String - - Name of the package to execute +- - `refresh_after_update` + - Boolean + - Whether the model should be refreshed after the update -- - `parameters` +- - `tables` - Sequence - - Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null. + - The tables to be exported to Power BI. See [\_](#jobsnametaskspower_bi_tasktables). + +- - `warehouse_id` + - String + - The SQL warehouse ID to use as the Power BI data source ::: -### jobs._name_.tasks.run_job_task +### jobs._name_.tasks.power_bi_task.power_bi_model **`Type: Map`** -The task triggers another job when the `run_job_task` field is present. +The semantic model to update @@ -4648,54 +4653,34 @@ The task triggers another job when the `run_job_task` field is present. - Type - Description -- - `dbt_commands` - - Sequence - - An array of commands to execute for jobs with the dbt task, for example `"dbt_commands": ["dbt deps", "dbt seed", "dbt deps", "dbt seed", "dbt run"]` - -- - `jar_params` - - Sequence - - A list of parameters for jobs with Spark JAR tasks, for example `"jar_params": ["john doe", "35"]`. The parameters are used to invoke the main function of the main class specified in the Spark JAR task. If not specified upon `run-now`, it defaults to an empty list. jar_params cannot be specified in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. - -- - `job_id` - - Integer - - ID of the job to trigger. - -- - `job_parameters` - - Map - - Job-level parameters used to trigger the job. - -- - `notebook_params` - - Map - - A map from keys to values for jobs with notebook task, for example `"notebook_params": {"name": "john doe", "age": "35"}`. The map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function. If not specified upon `run-now`, the triggered run uses the job’s base parameters. notebook_params cannot be specified in conjunction with jar_params. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. - -- - `pipeline_params` - - Map - - Controls whether the pipeline should perform a full refresh. See [\_](#jobsnametasksrun_job_taskpipeline_params). +- - `authentication_method` + - String + - How the published Power BI model authenticates to Databricks -- - `python_named_params` - - Map - - +- - `model_name` + - String + - The name of the Power BI model -- - `python_params` - - Sequence - - A list of parameters for jobs with Python tasks, for example `"python_params": ["john doe", "35"]`. The parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. Important These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. +- - `overwrite_existing` + - Boolean + - Whether to overwrite existing Power BI models -- - `spark_submit_params` - - Sequence - - A list of parameters for jobs with spark submit task, for example `"spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"]`. The parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs Important These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. +- - `storage_mode` + - String + - The default storage mode of the Power BI model -- - `sql_params` - - Map - - A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters. +- - `workspace_name` + - String + - The name of the Power BI workspace of the model ::: -### jobs._name_.tasks.run_job_task.pipeline_params +### jobs._name_.tasks.power_bi_task.tables -**`Type: Map`** +**`Type: Sequence`** -Controls whether the pipeline should perform a full refresh +The tables to be exported to Power BI @@ -4705,14 +4690,137 @@ Controls whether the pipeline should perform a full refresh - Type - Description -- - `full_refresh` - - Boolean - - If true, triggers a full refresh on the delta live table. +- - `catalog` + - String + - The catalog name in Databricks + +- - `name` + - String + - The table name in Databricks + +- - `schema` + - String + - The schema name in Databricks + +- - `storage_mode` + - String + - The Power BI storage mode of the table ::: -### jobs._name_.tasks.spark_jar_task +### jobs._name_.tasks.python_wheel_task + +**`Type: Map`** + +The task runs a Python wheel when the `python_wheel_task` field is present. + + + +:::list-table + +- - Key + - Type + - Description + +- - `entry_point` + - String + - Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()` + +- - `named_parameters` + - Map + - Command-line parameters passed to Python wheel task in the form of `["--name=task", "--data=dbfs:/path/to/data.json"]`. Leave it empty if `parameters` is not null. + +- - `package_name` + - String + - Name of the package to execute + +- - `parameters` + - Sequence + - Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null. + +::: + + +### jobs._name_.tasks.run_job_task + +**`Type: Map`** + +The task triggers another job when the `run_job_task` field is present. + + + +:::list-table + +- - Key + - Type + - Description + +- - `dbt_commands` + - Sequence + - This field is deprecated + +- - `jar_params` + - Sequence + - This field is deprecated + +- - `job_id` + - Integer + - ID of the job to trigger. + +- - `job_parameters` + - Map + - Job-level parameters used to trigger the job. + +- - `notebook_params` + - Map + - This field is deprecated + +- - `pipeline_params` + - Map + - Controls whether the pipeline should perform a full refresh. See [\_](#jobsnametasksrun_job_taskpipeline_params). + +- - `python_named_params` + - Map + - This field is deprecated + +- - `python_params` + - Sequence + - This field is deprecated + +- - `spark_submit_params` + - Sequence + - This field is deprecated + +- - `sql_params` + - Map + - This field is deprecated + +::: + + +### jobs._name_.tasks.run_job_task.pipeline_params + +**`Type: Map`** + +Controls whether the pipeline should perform a full refresh + + + +:::list-table + +- - Key + - Type + - Description + +- - `full_refresh` + - Boolean + - If true, triggers a full refresh on the delta live table. + +::: + + +### jobs._name_.tasks.spark_jar_task **`Type: Map`** @@ -4728,7 +4836,7 @@ The task runs a JAR when the `spark_jar_task` field is present. - - `jar_uri` - String - - Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create. + - This field is deprecated - - `main_class_name` - String @@ -4740,7 +4848,7 @@ The task runs a JAR when the `spark_jar_task` field is present. - - `run_as_repl` - Boolean - - Deprecated. A value of `false` is no longer supported. + - This field is deprecated ::: @@ -5175,7 +5283,7 @@ A configuration to trigger a run when certain conditions are met. The default be - - `table` - Map - - Old table trigger settings name. Deprecated in favor of `table_update`. See [\_](#jobsnametriggertable). + - This field is deprecated - - `table_update` - Map @@ -5238,39 +5346,6 @@ Periodic trigger settings. ::: -### jobs._name_.trigger.table - -**`Type: Map`** - -Old table trigger settings name. Deprecated in favor of `table_update`. - - - -:::list-table - -- - Key - - Type - - Description - -- - `condition` - - String - - The table(s) condition based on which to trigger a job run. - -- - `min_time_between_triggers_seconds` - - Integer - - If set, the trigger starts a run only after the specified amount of time has passed since the last time the trigger fired. The minimum allowed value is 60 seconds. - -- - `table_names` - - Sequence - - A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`. - -- - `wait_after_last_change_seconds` - - Integer - - If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds. - -::: - - ### jobs._name_.trigger.table_update **`Type: Map`** @@ -5470,7 +5545,7 @@ model_serving_endpoints: - - `ai_gateway` - Map - - The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. See [\_](#model_serving_endpointsnameai_gateway). + - The AI Gateway configuration for the serving endpoint. NOTE: External model, provisioned throughput, and pay-per-token endpoints are fully supported; agent endpoints currently only support inference tables. See [\_](#model_serving_endpointsnameai_gateway). - - `budget_policy_id` - String @@ -5490,7 +5565,7 @@ model_serving_endpoints: - - `rate_limits` - Sequence - - Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. See [\_](#model_serving_endpointsnamerate_limits). + - This field is deprecated - - `route_optimized` - Boolean @@ -5531,7 +5606,7 @@ resources: **`Type: Map`** -The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. +The AI Gateway configuration for the serving endpoint. NOTE: External model, provisioned throughput, and pay-per-token endpoints are fully supported; agent endpoints currently only support inference tables. @@ -5925,7 +6000,7 @@ The list of served entities under the serving endpoint config. - - `workload_size` - String - - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. + - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. - - `workload_type` - String @@ -6404,7 +6479,7 @@ PaLM Config. Only required if the provider is 'palm'. - - `workload_size` - String - - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. + - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. - - `workload_type` - String @@ -6475,48 +6550,19 @@ The list of routes that define traffic to each served entity. - - `group_name` - String - - The name of the group that has the permission set in level. + - - - `level` - String - - The allowed permission for user, group, service principal defined for this permission. + - - - `service_principal_name` - String - - The name of the service principal that has the permission set in level. + - - - `user_name` - String - - The name of the user that has the permission set in level. - -::: - - -### model_serving_endpoints._name_.rate_limits - -**`Type: Sequence`** - -Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. - - - -:::list-table - -- - Key - - Type - - Description - -- - `calls` - - Integer - - Used to specify how many calls are allowed for a key within the renewal_period. - -- - `key` - - String - - Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. - -- - `renewal_period` - - String - - Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported. + - ::: @@ -6565,22 +6611,10 @@ models: - Type - Description -- - `creation_timestamp` - - Integer - - Timestamp recorded when this `registered_model` was created. - - - `description` - String - Description of this `registered_model`. -- - `last_updated_timestamp` - - Integer - - Timestamp recorded when metadata for this `registered_model` was last updated. - -- - `latest_versions` - - Sequence - - Collection of latest model versions for each stage. Only contains models with current `READY` status. See [\_](#modelsnamelatest_versions). - - - `name` - String - Unique name for the model. @@ -6593,105 +6627,6 @@ models: - Sequence - Tags: Additional metadata key-value pairs for this `registered_model`. See [\_](#modelsnametags). -- - `user_id` - - String - - User that created this `registered_model` - -::: - - -### models._name_.latest_versions - -**`Type: Sequence`** - -Collection of latest model versions for each stage. -Only contains models with current `READY` status. - - - -:::list-table - -- - Key - - Type - - Description - -- - `creation_timestamp` - - Integer - - Timestamp recorded when this `model_version` was created. - -- - `current_stage` - - String - - Current stage for this `model_version`. - -- - `description` - - String - - Description of this `model_version`. - -- - `last_updated_timestamp` - - Integer - - Timestamp recorded when metadata for this `model_version` was last updated. - -- - `name` - - String - - Unique name of the model - -- - `run_id` - - String - - MLflow run ID used when creating `model_version`, if `source` was generated by an experiment run stored in MLflow tracking server. - -- - `run_link` - - String - - Run Link: Direct link to the run that generated this version - -- - `source` - - String - - URI indicating the location of the source model artifacts, used when creating `model_version` - -- - `status` - - String - - Current status of `model_version` - -- - `status_message` - - String - - Details on current `status`, if it is pending or failed. - -- - `tags` - - Sequence - - Tags: Additional metadata key-value pairs for this `model_version`. See [\_](#modelsnamelatest_versionstags). - -- - `user_id` - - String - - User that created this `model_version`. - -- - `version` - - String - - Model's version number. - -::: - - -### models._name_.latest_versions.tags - -**`Type: Sequence`** - -Tags: Additional metadata key-value pairs for this `model_version`. - - - -:::list-table - -- - Key - - Type - - Description - -- - `key` - - String - - The tag key. - -- - `value` - - String - - The tag value. - ::: @@ -6711,19 +6646,19 @@ Tags: Additional metadata key-value pairs for this `model_version`. - - `group_name` - String - - The name of the group that has the permission set in level. + - - - `level` - String - - The allowed permission for user, group, service principal defined for this permission. + - - - `service_principal_name` - String - - The name of the service principal that has the permission set in level. + - - - `user_name` - String - - The name of the user that has the permission set in level. + - ::: @@ -6882,7 +6817,7 @@ pipelines: - - `trigger` - Map - - Which pipeline trigger to use. Deprecated: Use `continuous` instead. See [\_](#pipelinesnametrigger). + - Use continuous instead ::: @@ -6941,7 +6876,7 @@ Cluster settings for this pipeline deployment. - - `cluster_log_conf` - Map - - The configuration for delivering spark logs to a long-term storage destination. Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. . See [\_](#pipelinesnameclusterscluster_log_conf). + - The configuration for delivering spark logs to a long-term storage destination. Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [\_](#pipelinesnameclusterscluster_log_conf). - - `custom_tags` - Map @@ -6977,7 +6912,7 @@ Cluster settings for this pipeline deployment. - - `node_type_id` - String - - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. + - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. - - `num_workers` - Integer @@ -6989,7 +6924,7 @@ Cluster settings for this pipeline deployment. - - `spark_conf` - Map - - An object containing a set of optional, user-specified Spark configuration key-value pairs. See :method:clusters/create for more details. + - An object containing a set of optional, user-specified Spark configuration key-value pairs. See :method:clusters/create for more details. - - `spark_env_vars` - Map @@ -7027,7 +6962,7 @@ Note: autoscaling works best with DB runtime versions 3.0 or later. - - `mode` - String - - Databricks Enhanced Autoscaling optimizes cluster utilization by automatically allocating cluster resources based on workload volume, with minimal impact to the data processing latency of your pipelines. Enhanced Autoscaling is available for `updates` clusters only. The legacy autoscaling feature is used for `maintenance` clusters. + - Databricks Enhanced Autoscaling optimizes cluster utilization by automatically allocating cluster resources based on workload volume, with minimal impact to the data processing latency of your pipelines. Enhanced Autoscaling is available for `updates` clusters only. The legacy autoscaling feature is used for `maintenance` clusters. ::: @@ -7158,7 +7093,6 @@ Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. - @@ -7308,7 +7242,7 @@ If not specified at cluster creation, a set of default values will be used. - - `use_preemptible_executors` - Boolean - - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the 'availability' field instead. + - This field is deprecated - - `zone_id` - String @@ -7337,7 +7271,7 @@ The configuration for storing init scripts. Any number of destinations can be sp - - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`. See [\_](#pipelinesnameclustersinit_scriptsdbfs). + - This field is deprecated - - `file` - Map @@ -7383,28 +7317,6 @@ Contains the Azure Data Lake Storage destination path ::: -### pipelines._name_.clusters.init_scripts.dbfs - -**`Type: Map`** - -destination needs to be provided. e.g. -`{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }` - - - -:::list-table - -- - Key - - Type - - Description - -- - `destination` - - String - - dbfs destination, e.g. `dbfs:/my/path` - -::: - - ### pipelines._name_.clusters.init_scripts.file **`Type: Map`** @@ -7636,7 +7548,7 @@ The definition of a gateway pipeline to support change data capture. - - `connection_id` - String - - [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. + - This field is deprecated - - `connection_name` - String @@ -7648,7 +7560,7 @@ The definition of a gateway pipeline to support change data capture. - - `gateway_storage_name` - String - - Optional. The Unity Catalog-compatible name for the gateway storage location. This is the destination to use for the data that is extracted by the gateway. Delta Live Tables system will automatically create the storage location under the catalog and schema. + - Optional. The Unity Catalog-compatible name for the gateway storage location. This is the destination to use for the data that is extracted by the gateway. Delta Live Tables system will automatically create the storage location under the catalog and schema. - - `gateway_storage_schema` - String @@ -7770,6 +7682,14 @@ Configuration settings to control the ingestion of tables. These settings overri - Type - Description +- - `exclude_columns` + - Sequence + - A list of column names to be excluded for the ingestion. When not specified, include_columns fully controls what columns to be ingested. When specified, all other columns including future ones will be automatically included for ingestion. This field in mutually exclusive with `include_columns`. + +- - `include_columns` + - Sequence + - A list of column names to be included for the ingestion. When not specified, all columns except ones in exclude_columns will be included. Future columns will be automatically included. When specified, all other future columns will be automatically excluded from ingestion. This field in mutually exclusive with `exclude_columns`. + - - `primary_keys` - Sequence - The primary key of the table used to apply changes. @@ -7840,6 +7760,14 @@ Configuration settings to control the ingestion of tables. These settings are ap - Type - Description +- - `exclude_columns` + - Sequence + - A list of column names to be excluded for the ingestion. When not specified, include_columns fully controls what columns to be ingested. When specified, all other columns including future ones will be automatically included for ingestion. This field in mutually exclusive with `include_columns`. + +- - `include_columns` + - Sequence + - A list of column names to be included for the ingestion. When not specified, all columns except ones in exclude_columns will be included. Future columns will be automatically included. When specified, all other future columns will be automatically excluded from ingestion. This field in mutually exclusive with `exclude_columns`. + - - `primary_keys` - Sequence - The primary key of the table used to apply changes. @@ -7918,6 +7846,14 @@ Configuration settings to control the ingestion of tables. These settings overri - Type - Description +- - `exclude_columns` + - Sequence + - A list of column names to be excluded for the ingestion. When not specified, include_columns fully controls what columns to be ingested. When specified, all other columns including future ones will be automatically included for ingestion. This field in mutually exclusive with `include_columns`. + +- - `include_columns` + - Sequence + - A list of column names to be included for the ingestion. When not specified, all columns except ones in exclude_columns will be included. Future columns will be automatically included. When specified, all other future columns will be automatically excluded from ingestion. This field in mutually exclusive with `exclude_columns`. + - - `primary_keys` - Sequence - The primary key of the table used to apply changes. @@ -7951,6 +7887,14 @@ Configuration settings to control the ingestion of tables. These settings are ap - Type - Description +- - `exclude_columns` + - Sequence + - A list of column names to be excluded for the ingestion. When not specified, include_columns fully controls what columns to be ingested. When specified, all other columns including future ones will be automatically included for ingestion. This field in mutually exclusive with `include_columns`. + +- - `include_columns` + - Sequence + - A list of column names to be included for the ingestion. When not specified, all columns except ones in exclude_columns will be included. Future columns will be automatically included. When specified, all other future columns will be automatically excluded from ingestion. This field in mutually exclusive with `exclude_columns`. + - - `primary_keys` - Sequence - The primary key of the table used to apply changes. @@ -7986,23 +7930,23 @@ Libraries or code needed by this deployment. - - `file` - Map - - The path to a file that defines a pipeline and is stored in the Databricks Repos. . See [\_](#pipelinesnamelibrariesfile). + - The path to a file that defines a pipeline and is stored in the Databricks Repos. See [\_](#pipelinesnamelibrariesfile). - - `jar` - String - - URI of the jar to be installed. Currently only DBFS is supported. + - URI of the jar to be installed. Currently only DBFS is supported. - - `maven` - Map - - Specification of a maven library to be installed. . See [\_](#pipelinesnamelibrariesmaven). + - Specification of a maven library to be installed. See [\_](#pipelinesnamelibrariesmaven). - - `notebook` - Map - - The path to a notebook that defines a pipeline and is stored in the Databricks workspace. . See [\_](#pipelinesnamelibrariesnotebook). + - The path to a notebook that defines a pipeline and is stored in the Databricks workspace. See [\_](#pipelinesnamelibrariesnotebook). - - `whl` - String - - URI of the whl to be installed. + - This field is deprecated ::: @@ -8012,7 +7956,6 @@ Libraries or code needed by this deployment. **`Type: Map`** The path to a file that defines a pipeline and is stored in the Databricks Repos. - @@ -8024,7 +7967,7 @@ The path to a file that defines a pipeline and is stored in the Databricks Repos - - `path` - String - - The absolute path of the file. + - The absolute path of the source code. ::: @@ -8034,7 +7977,6 @@ The path to a file that defines a pipeline and is stored in the Databricks Repos **`Type: Map`** Specification of a maven library to be installed. - @@ -8064,7 +8006,6 @@ Specification of a maven library to be installed. **`Type: Map`** The path to a notebook that defines a pipeline and is stored in the Databricks workspace. - @@ -8076,7 +8017,7 @@ The path to a notebook that defines a pipeline and is stored in the Databricks w - - `path` - String - - The absolute path of the notebook. + - The absolute path of the source code. ::: @@ -8097,11 +8038,11 @@ List of notification settings for this pipeline. - - `alerts` - Sequence - - A list of alerts that trigger the sending of notifications to the configured destinations. The supported alerts are: * `on-update-success`: A pipeline update completes successfully. * `on-update-failure`: Each time a pipeline update fails. * `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error. * `on-flow-failure`: A single data flow fails. + - A list of alerts that trigger the sending of notifications to the configured destinations. The supported alerts are: * `on-update-success`: A pipeline update completes successfully. * `on-update-failure`: Each time a pipeline update fails. * `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error. * `on-flow-failure`: A single data flow fails. - - `email_recipients` - Sequence - - A list of email addresses notified when a configured alert is triggered. + - A list of email addresses notified when a configured alert is triggered. ::: @@ -8122,19 +8063,19 @@ List of notification settings for this pipeline. - - `group_name` - String - - The name of the group that has the permission set in level. + - - - `level` - String - - The allowed permission for user, group, service principal defined for this permission. + - - - `service_principal_name` - String - - The name of the service principal that has the permission set in level. + - - - `user_name` - String - - The name of the user that has the permission set in level. + - ::: @@ -8203,63 +8144,6 @@ Only `user_name` or `service_principal_name` can be specified. If both are speci ::: -### pipelines._name_.trigger - -**`Type: Map`** - -Which pipeline trigger to use. Deprecated: Use `continuous` instead. - - - -:::list-table - -- - Key - - Type - - Description - -- - `cron` - - Map - - See [\_](#pipelinesnametriggercron). - -- - `manual` - - Map - - - -::: - - -### pipelines._name_.trigger.cron - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `quartz_cron_schedule` - - String - - - -- - `timezone_id` - - String - - - -::: - - -### pipelines._name_.trigger.manual - -**`Type: Map`** - - - - ## quality_monitors **`Type: Map`** @@ -8800,6 +8684,77 @@ resources: ::: +## secret_scopes + +**`Type: Map`** + + + +```yaml +secret_scopes: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `backend_type` + - String + - The backend type the scope will be created with. If not specified, will default to `DATABRICKS` + +- - `keyvault_metadata` + - Map + - The metadata for the secret scope if the `backend_type` is `AZURE_KEYVAULT`. See [\_](#secret_scopesnamekeyvault_metadata). + +- - `name` + - String + - Scope name requested by the user. Scope names are unique. + +- - `permissions` + - Sequence + - The permissions to apply to the secret scope. Permissions are managed via secret scope ACLs. See [\_](#secret_scopesnamepermissions). + +::: + + +### secret_scopes._name_.permissions + +**`Type: Sequence`** + +The permissions to apply to the secret scope. Permissions are managed via secret scope ACLs. + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. This field translates to a `principal` field in secret scope ACL. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The application ID of an active service principal. This field translates to a `principal` field in secret scope ACL. + +- - `user_name` + - String + - The name of the user that has the permission set in level. This field translates to a `principal` field in secret scope ACL. + +::: + + ## volumes **`Type: Map`** @@ -8849,7 +8804,7 @@ volumes: - - `volume_type` - String - - + - The type of the volume. An external volume is located in the specified external location. A managed volume is located in the default location which is specified by the parent schema, or the parent catalog, or the Metastore. [Learn more](https://docs.databricks.com/aws/en/volumes/managed-vs-external) ::: diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index bc445055d5..b9955647a9 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -199,6 +199,9 @@ github.com/databricks/cli/bundle/config.Resources: The schema definitions for the bundle, where each key is the name of the schema. "markdown_description": |- The schema definitions for the bundle, where each key is the name of the schema. See [\_](/dev-tools/bundles/resources.md#schemas). + "secret_scopes": + "description": |- + PLACEHOLDER "volumes": "description": |- The volume definitions for the bundle, where each key is the name of the volume. @@ -532,6 +535,32 @@ github.com/databricks/cli/bundle/config/resources.PipelinePermission: "user_name": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.SecretScope: + "backend_type": + "description": |- + The backend type the scope will be created with. If not specified, will default to `DATABRICKS` + "keyvault_metadata": + "description": |- + The metadata for the secret scope if the `backend_type` is `AZURE_KEYVAULT` + "name": + "description": |- + Scope name requested by the user. Scope names are unique. + "permissions": + "description": |- + The permissions to apply to the secret scope. Permissions are managed via secret scope ACLs. +github.com/databricks/cli/bundle/config/resources.SecretScopePermission: + "group_name": + "description": |- + The name of the group that has the permission set in level. This field translates to a `principal` field in secret scope ACL. + "level": + "description": |- + The allowed permission for user, group, service principal defined for this permission. + "service_principal_name": + "description": |- + The application ID of an active service principal. This field translates to a `principal` field in secret scope ACL. + "user_name": + "description": |- + The name of the user that has the permission set in level. This field translates to a `principal` field in secret scope ACL. github.com/databricks/cli/bundle/config/variable.Lookup: "alert": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 5bba842541..40e75055f0 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -45,6 +45,15 @@ github.com/databricks/cli/bundle/config/resources.AppPermissionLevel: CAN_MANAGE - |- CAN_USE +github.com/databricks/cli/bundle/config/resources.SecretScopePermissionLevel: + "_": + "enum": + - |- + READ + - |- + WRITE + - |- + MANAGE github.com/databricks/cli/bundle/config/resources.Cluster: "_": "markdown_description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index ed7ab6bab3..8d305809e2 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1304,6 +1304,88 @@ } ] }, + "resources.SecretScope": { + "oneOf": [ + { + "type": "object", + "properties": { + "backend_type": { + "description": "The backend type the scope will be created with. If not specified, will default to `DATABRICKS`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/workspace.ScopeBackendType" + }, + "keyvault_metadata": { + "description": "The metadata for the secret scope if the `backend_type` is `AZURE_KEYVAULT`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/workspace.AzureKeyVaultSecretScopeMetadata" + }, + "name": { + "description": "Scope name requested by the user. Scope names are unique.", + "$ref": "#/$defs/string" + }, + "permissions": { + "description": "The permissions to apply to the secret scope. Permissions are managed via secret scope ACLs.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.SecretScopePermission" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SecretScopePermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "description": "The name of the group that has the permission set in level. This field translates to a `principal` field in secret scope ACL.", + "$ref": "#/$defs/string" + }, + "level": { + "description": "The allowed permission for user, group, service principal defined for this permission.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SecretScopePermissionLevel" + }, + "service_principal_name": { + "description": "The application ID of an active service principal. This field translates to a `principal` field in secret scope ACL.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The name of the user that has the permission set in level. This field translates to a `principal` field in secret scope ACL.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SecretScopePermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "READ", + "WRITE", + "MANAGE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Volume": { "oneOf": [ { @@ -1843,6 +1925,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Schema", "markdownDescription": "The schema definitions for the bundle, where each key is the name of the schema. See [schemas](https://docs.databricks.com/dev-tools/bundles/resources.html#schemas)." }, + "secret_scopes": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SecretScope" + }, "volumes": { "description": "The volume definitions for the bundle, where each key is the name of the volume.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume", @@ -7343,6 +7428,33 @@ "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] + }, + "workspace.AzureKeyVaultSecretScopeMetadata": { + "oneOf": [ + { + "type": "object", + "properties": { + "dns_name": { + "$ref": "#/$defs/string" + }, + "resource_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "dns_name", + "resource_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "workspace.ScopeBackendType": { + "type": "string" } } } @@ -7563,6 +7675,20 @@ } ] }, + "resources.SecretScope": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SecretScope" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Volume": { "oneOf": [ { @@ -7826,6 +7952,20 @@ "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] + }, + "resources.SecretScopePermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SecretScopePermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] } }, "config.ArtifactFile": {