update acceptance tests

Author: fjakobs

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/.vscode/settings.json

@@ -8,7 +8,7 @@
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
-    "python.analysis.extraPaths": ["assets/etl_pipeline"],
+    "python.analysis.extraPaths": ["resources/my_lakeflow_pipelines_pipeline"],
     "files.exclude": {
         "**/*.egg-info": true,
         "**/__pycache__": true,

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/README.md

@@ -1,6 +1,6 @@
 # my_lakeflow_pipelines
 
-The 'my_lakeflow_pipelines' project was generated by using the Lakeflow template.
+The 'my_lakeflow_pipelines' project was generated by using the Lakeflow Pipelines template.
 
 ## Setup
 

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/databricks.yml

@@ -36,13 +36,11 @@ targets:
     mode: production
     workspace:
       host: [DATABRICKS_URL]
-      # We explicitly specify /Workspace/Users/[USERNAME] to make sure we only have a single copy.
+      # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy.
       root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}
     permissions:
       - user_name: [USERNAME]
         level: CAN_MANAGE
-    run_as:
-      user_name: [USERNAME]
     variables:
       catalog: main
       schema: default

libs/template/templates/lakeflow-pipelines/databricks_template_schema.json

@@ -3,7 +3,7 @@
     "properties": {
         "project_name": {
             "type": "string",
-            "default": "my_project",
+            "default": "my_lakeflow_project",
             "description": "Please provide the following details to tailor the template to your preferences.\n\nUnique name for this project\nproject_name",
             "order": 1,
             "pattern": "^[a-z0-9_]+$",
@@ -45,7 +45,7 @@
         "language": {
             "type": "string",
             "default": "python",
-            "description": "\nLanguage for this project:\nlanguage",
+            "description": "\nInitial language for this project:\nlanguage",
             "enum": [
                 "python",
                 "sql"

libs/template/templates/lakeflow-pipelines/library/variables.tmpl

@@ -1,5 +1,5 @@
 {{- define `table_suffix` -}}
-    {{ (regexp `^_+|_+$`).ReplaceAllString ((regexp `_+`).ReplaceAllString .project_name `_`) `` }}
+    {{ .project_name }}{{/* {{ (regexp `^_+|_+$`).ReplaceAllString ((regexp `_+`).ReplaceAllString .project_name `_`) `` }} */}}
 {{- end }}
 
 {{- define `pipeline_name` -}}

libs/template/templates/lakeflow-pipelines/template/__preamble.tmpl

@@ -7,10 +7,10 @@ This file only contains template directives; it is skipped for the actual output
 {{$isSQL := eq .language "sql"}}
 
 {{if $isSQL}}
-  {{skip "{{.project_name}}/resources/{{template `pipeline_name` .}}/utilities/utils.py"}}
-  {{skip "{{.project_name}}/resources/{{template `pipeline_name` .}}/transformations/sample_zones_{{template `table_suffix` .}}.py"}}
-  {{skip "{{.project_name}}/resources/{{template `pipeline_name` .}}/transformations/sample_trips_{{template `table_suffix` .}}.py"}}
+  {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/utilities/utils.py"}}
+  {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.py"}}
+  {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.py"}}
 {{else}}
-  {{skip "{{.project_name}}/resources/{{template `pipeline_name` .}}/transformations/sample_zones_{{template `table_suffix` .}}.sql"}}
-  {{skip "{{.project_name}}/resources/{{template `pipeline_name` .}}/transformations/sample_trips_{{template `table_suffix` .}}.sql"}}
+  {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_zones_{{.project_name}}.sql"}}
+  {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline/transformations/sample_trips_{{.project_name}}.sql"}}
 {{end}}

libs/template/templates/lakeflow-pipelines/template/{{.project_name}}/.vscode/settings.json.tmpl

@@ -9,7 +9,7 @@
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
     {{- /* Unfortunately extraPaths doesn't support globs!! See: https://github.com/microsoft/pylance-release/issues/973 */}}
-    "python.analysis.extraPaths": ["assets/etl_pipeline"],
+    "python.analysis.extraPaths": ["resources/{{.project_name}}_pipeline"],
     "files.exclude": {
         "**/*.egg-info": true,
         "**/__pycache__": true,

libs/template/templates/lakeflow-pipelines/template/{{.project_name}}/README.md.tmpl

@@ -1,6 +1,6 @@
 # {{.project_name}}
 
-The '{{.project_name}}' project was generated by using the Lakeflow template.
+The '{{.project_name}}' project was generated by using the Lakeflow Pipelines template.
 
 ## Setup
 

libs/template/templates/lakeflow-pipelines/template/{{.project_name}}/databricks.yml.tmpl

@@ -36,13 +36,11 @@ targets:
     mode: production
     workspace:
       host: {{workspace_host}}
-      # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy.
+      # We explicitly deploy to /Workspace/Users/{{user_name}} to make sure we only have a single copy.
       root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target}
     permissions:
       - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}
         level: CAN_MANAGE
-    run_as:
-      {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}
     variables:
       catalog: {{.default_catalog}}
       schema: {{template `prod_schema` .}}

libs/template/templates/lakeflow-pipelines/template/{{.project_name}}/resources/{{.project_name}}_pipeline/README.md.tmpl

@@ -1,11 +1,12 @@
 {{- if (eq .language "python") -}}
+
 # {{template `pipeline_name` .}}
 
 This folder defines all source code for the {{template `pipeline_name` .}} pipeline:
 
 - `explorations`: Ad-hoc notebooks used to explore the data processed by this pipeline.
 - `transformations`: All dataset definitions and transformations.
-- `utilities`: Utility functions and Python modules used in this pipeline.
+- `utilities` (optional): Utility functions and Python modules used in this pipeline.
 - `data_sources` (optional): View definitions describing the source data for this pipeline.
 
 ## Getting Started
@@ -21,7 +22,9 @@ To get started, go to the `transformations` folder -- most of the relevant sourc
 * Use `Schedule` to run the pipeline on a schedule!
 
 For more tutorials and reference material, see https://docs.databricks.com/dlt.
+
 {{- else -}}
+
 # {{template `pipeline_name` .}}
 
 This folder defines all source code for the '{{template `pipeline_name` .}}' pipeline:
@@ -43,4 +46,5 @@ To get started, go to the \`transformations\` folder -- most of the relevant sou
 * Use \`Schedule\` to run the pipeline on a schedule!
 
 For more tutorials and reference material, see https://docs.databricks.com/dlt.
+
 {{- end -}}