feat(dbt-fal): adapt to new 1.5 dbt changes (#814)

* update poetry deps

* chore: adapt to new flags reading and introduce AdapterConfig

* tests: fix dbt version to test

* tests: fix dbt version to test

* change OVERRIDE_PROPERTIES for easier logic

* send flags to environment-run models

* remove deprecated target-path

* Set target path for integartion tests based on temp dir

* Make _isolated_runner receive only named args

* remove redshift for now

* use 1.5 release candidate

* formatting

* use adapt to new model.refs format and new model versioning

* use 1.5.0

* Poetry lock updated

---------

Co-authored-by: chamini2 <chamini2@users.noreply.github.com>

Author: chamini2

.github/workflows/test_integration_adapter.yml

@@ -28,14 +28,16 @@ jobs:
         profile:
           - postgres
           - bigquery
-          # - duckdb
           - snowflake
-          - redshift
-          - athena
-          - trino
-          # - sqlserver # does not support dbt 1.4 yet
+          # TODO: redshift had a bigger change between 1.4 and 1.5 and will be addressed separately
+          # - redshift
+          # TODO: enable as 1.5 becomes available for following adapters
+          # - athena
+          # - trino
+          # - duckdb
+          # - sqlserver
         dbt_version:
-          - "1.4.*"
+          - "1.5.0"
         python:
           # - "3.7"
           - "3.8"
@@ -46,8 +48,8 @@ jobs:
           - profile: snowflake
             teleport: true
             cloud: true
-          - profile: redshift
-            cloud: true
+          # - profile: redshift
+          #   cloud: true
           - profile: bigquery
             cloud: true
 

projects/adapter/integration_tests/features/steps/fal_adapter_steps.py

@@ -10,6 +10,10 @@
 import re
 
 
+def target_path(context):
+    return str(Path(context.temp_dir.name) / "target")
+
+
 @when("the following shell command is invoked")
 def run_command_step(context):
     context.exc = None
@@ -46,7 +50,7 @@ def set_project_folder(context, project: str):
     context.base_dir = str(project_path)
     context.temp_dir = tempfile.TemporaryDirectory()
     context.project_name = _load_dbt_project_file(context)["name"]
-    os.environ["temp_dir"] = context.temp_dir.name
+    os.environ["DBT_TARGET_PATH"] = target_path(context)
 
 
 @then("the following models are calculated in order")
@@ -93,9 +97,7 @@ def _get_dated_dbt_models(context):
 
 
 def _load_dbt_result_file(context):
-    with open(
-        os.path.join(context.temp_dir.name, "target", "run_results.json")
-    ) as stream:
+    with open(os.path.join(target_path(context), "run_results.json")) as stream:
         return json.load(stream)["results"]
 
 
@@ -105,10 +107,9 @@ def _load_dbt_project_file(context):
 
 
 def _load_target_run_model(context, model_name: str, file_ext: str):
-
     # TODO: we should use fal to find these files from fal reading the dbt_project.yml and making it easily available
     models_dir: Path = (
-        Path(context.temp_dir.name) / "target" / "run" / context.project_name / "models"
+        Path(target_path(context)) / "run" / context.project_name / "models"
     )
 
     found_model_files = list(models_dir.rglob(f"{model_name}.{file_ext}"))
@@ -121,7 +122,6 @@ def _load_target_run_model(context, model_name: str, file_ext: str):
 def _replace_vars(context, msg):
     return (
         msg.replace("$baseDir", context.base_dir)
-        .replace("$tempDir", str(context.temp_dir.name))
         .replace("$profilesDir", _get_profiles_dir(context))
         .replace("$profile", _get_profile(context))
     )
@@ -140,7 +140,7 @@ def _get_profile(context) -> str:
         "duckdb",
         "athena",
         "trino",
-        "sqlserver"
+        "sqlserver",
     ]
     profile = context.config.userdata.get("profile", "postgres")
     if profile not in available_profiles:

projects/adapter/integration_tests/projects/env_project/dbt_project.yml

@@ -8,8 +8,6 @@ seed-paths: ["data"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
-target-path: "{{ env_var('temp_dir') }}/target"
-
 vars:
   fal-scripts-path: "fal_scripts"
 

projects/adapter/integration_tests/projects/simple_project/dbt_project.yml

@@ -8,10 +8,10 @@ seed-paths: ["data"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
-target-path: "{{ env_var('temp_dir') }}/target"
-
 models:
   +schema: custom
+  # All serverless should run on M machines by default
+  +fal_machine: "M"
   simple_test:
     python:
       +materialized: table

projects/adapter/integration_tests/projects/source_freshness/dbt_project.yml

@@ -8,8 +8,6 @@ seed-paths: ["data"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
 
-target-path: "{{ env_var('temp_dir') }}/target"
-
 vars:
   fal-scripts-path: "fal_scripts"