Feature/support dbt core1.7.x

.github/workflows/test_cli.yml

@@ -19,9 +19,9 @@ jobs:
           - "3.8"
           # - "3.9"
           - "3.10"
-          # - "3.11"
+          - "3.11"
         dbt:
-          - "1.5.*"
+          - "1.7.*"
 
     # Run only the latest commit pushed to PR
     concurrency:

.github/workflows/test_integration_adapter.yml

@@ -33,12 +33,12 @@ jobs:
           # - duckdb
           # - sqlserver
         dbt_version:
-          - "1.5.0"
+          - "1.7.8"
         python:
           - "3.8"
           - "3.9"
           - "3.10"
-          # - "3.11"
+          - "3.11"
         include:
           - profile: snowflake
             teleport: true

.github/workflows/test_integration_cli.yml

@@ -52,7 +52,7 @@ on:
         options:
           - "<ALL>"
           - "latest"
-          - "1.5.*"
+          - "1.7.*"
 
 jobs:
   matrix-adapter:
@@ -163,7 +163,7 @@ jobs:
         shell: python
         run: |
           OPTIONS = [
-            "1.5.*",
+            "1.7.*",
           ]
           OUTPUT = OPTIONS
 

projects/adapter/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dbt-fal"
-version = "1.5.10a0"
+version = "1.7.10a0"
 # name = "fal"
 # version = "0.9.4a0"
 description = "Run python scripts from any dbt project."
@@ -22,7 +22,7 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = "^3.8"
-dbt-core = ">=1.5,<=1.5.5"
+dbt-core = ">=1.6,<=1.7.8"
 pandas = "^1.3.4"
 posthog = "^1.4.5"
 "backports.functools_lru_cache" = "^1.6.4"
@@ -42,6 +42,10 @@ dill = "0.3.7"
 # dbt-fal
 sqlalchemy = "^1.4.41"
 
+# https://github.com/dbt-labs/dbt-core/issues/9566
+# TODO: Remove when this project advances to dbt-core 1.8.x
+protobuf = ">=4.0.0,<5"
+
 # Adapters
 
 ## snowflake

projects/adapter/src/dbt/adapters/fal/__version__.py

@@ -1 +1 @@
-version = '1.5.10a0'
+version = '1.7.10a0'

projects/adapter/src/dbt/adapters/fal_experimental/__version__.py

@@ -1 +1 @@
-version = '1.5.10a0'
+version = '1.7.10a0'

projects/adapter/src/fal/dbt/cli/fal_runner.py

@@ -14,8 +14,11 @@ def create_fal_dbt(
     args: argparse.Namespace, generated_models: Dict[str, Path] = {}
 ) -> FalDbt:
     real_state = None
+    real_defer_state = None
     if hasattr(args, "state") and args.state is not None:
         real_state = args.state
+    if hasattr(args, "real_defer_state") and args.defer_state is not None:
+        real_defer_state = args.state
 
     return FalDbt(
         args.project_dir,
@@ -25,6 +28,7 @@ def create_fal_dbt(
         args.selector,
         args.threads,
         real_state,
+        real_defer_state,
         args.target,
         args.vars,
         generated_models,

projects/adapter/src/fal/dbt/integration/parse.py

@@ -5,7 +5,7 @@
 from typing import Any, List, Dict, Optional, Union, TYPE_CHECKING
 
 from dbt.contracts.project import Project as ProjectContract
-from dbt.config import RuntimeConfig, Project
+from dbt.config import RuntimeConfig, Project, PartialProject
 from dbt.config.utils import parse_cli_vars as dbt_parse_cli_vars
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.results import RunResultsArtifact, FreshnessExecutionResultArtifact
@@ -47,7 +47,7 @@ class RuntimeArgs:
 
 
 def load_dbt_project_contract(project_dir: str) -> ProjectContract:
-    partial_project = Project.partial_load(project_dir)
+    partial_project = PartialProject.from_project_root(project_dir)
     contract = ProjectContract.from_dict(partial_project.project_dict)
     if not hasattr(contract, "model_paths") or contract.model_paths is None:
         setattr(contract, "model_paths", contract.source_paths)

projects/adapter/src/fal/dbt/integration/project.py

@@ -436,6 +436,7 @@ class CompileArgs:
     models: List[str]
     exclude: Tuple[str]
     state: Optional[Path]
+    defer_state: Optional[Path]
     single_threaded: Optional[bool]
 
 
@@ -453,6 +454,7 @@ def __init__(
         selector: Optional[str] = None,
         threads: Optional[int] = None,
         state: Optional[str] = None,
+        defer_state: Optional[str] = None,
         profile_target: Optional[str] = None,
         args_vars: str = "{}",
         generated_models: Dict[str, Path] = {},
@@ -485,9 +487,13 @@ def __init__(
         self.profiles_dir = flags.PROFILES_DIR
 
         self._state = None
+        self._defer_state = None
         if state is not None:
             self._state = Path(os.path.realpath(os.path.expanduser(state)))
 
+        if defer_state is not None:
+            self._defer_state = Path(os.path.realpath(os.path.expanduser(defer_state)))
+
         self.scripts_dir = parse.get_scripts_dir(self.project_dir, args_vars)
 
 
@@ -525,7 +531,7 @@ def __init__(
         # Necessary for manifest loading to not fail
         # dbt.tracking.initialize_tracking(self.profiles_dir)
 
-        args = CompileArgs(selector, select, select, exclude, self._state, None)
+        args = CompileArgs(selector, select, select, exclude, self._state, self._defer_state, None)
         self._compile_task = CompileTask(args, self._config, native_manifest)
 
         self._compile_task._runtime_initialize()
@@ -677,6 +683,7 @@ def _model(
     ) -> ManifestNode:
         # HACK: always setting node package as self.project_dir
         target_model: MaybeNonSource = self._manifest.native_manifest.resolve_ref(
+            None,
             target_model_name,
             target_package_name,
             None,

projects/adapter/tests/test_cli.py

@@ -67,7 +67,7 @@ def test_run_with_project_dir(capfd):
         # TODO: should it run without a run_results and no selection flags?
         cli([
             # fmt: off
-            "fal", "run",
+            "fal", "run", "--all",
             "--project-dir", tmp_dir,
             "--profiles-dir", profiles_dir,
             # fmt: on
@@ -96,7 +96,7 @@ def test_flow_run_with_project_dir(capfd):
         )
 
         executing_re = re.compile(
-            r": dbt run --threads 1 --project-dir [\w\d\/\-\_]+ --profiles-dir [\w\d\/\-\_]+tests/mock/mockProfile"
+            r": dbt run --threads 1 --project-dir [\w\d\/\-\_]+ --profiles-dir [\w\d\/\-\_\.]+tests/mock/mockProfile"
         )
         found = executing_re.findall(captured.out)
         # We run each model separately
@@ -118,7 +118,7 @@ def test_flow_run_with_project_dir_and_select(capfd):
         )
 
         executing_re = re.compile(
-            r": dbt run --threads 1 --project-dir [\w\/\-\_]+ --profiles-dir [\w\/\-\_]+tests/mock/mockProfile \--select|\--models model_with_before_scripts"
+            r": dbt run --threads 1 --project-dir [\w\/\-\_]+ --profiles-dir [\w\/\-\_\.]+tests/mock/mockProfile \--select|\--models model_with_before_scripts"
         )
         found = executing_re.findall(captured.out)
         assert len(found) == 1
@@ -144,7 +144,7 @@ def test_flow_run_with_defer(capfd):
         )
 
         executing_re = re.compile(
-            r": dbt run --threads 1 --project-dir [\w\/\-\_]+ --profiles-dir [\w\/\-\_]+tests/mock/mockProfile --defer --state [\w\/\-\_]+/target"
+            r": dbt run --threads 1 --project-dir [\w\/\-\_]+ --profiles-dir [\w\/\-\_\.]+tests/mock/mockProfile --defer --state [\w\/\-\_]+/target"
         )
         found = executing_re.findall(captured.out)
         # We run each model separately
@@ -166,7 +166,7 @@ def test_flow_run_with_full_refresh(capfd):
         )
 
         executing_re = re.compile(
-            r": dbt run --threads 1 --project-dir [\w\d\/\-\_]+ --profiles-dir [\w\d\/\-\_]+tests/mock/mockProfile --full-refresh"
+            r": dbt run --threads 1 --project-dir [\w\d\/\-\_]+ --profiles-dir [\w\d\/\-\_\.]+tests/mock/mockProfile --full-refresh"
         )
         found = executing_re.findall(captured.out)
         # We run each model separately
@@ -189,7 +189,7 @@ def test_flow_run_with_vars(capfd):
         )
 
         executing_re = re.compile(
-            r": dbt run --threads 1 --project-dir [\w\/\-\_]+ --profiles-dir [\w\/\-\_]+tests/mock/mockProfile --vars " + var_str
+            r": dbt run --threads 1 --project-dir [\w\/\-\_]+ --profiles-dir [\w\/\-\_\.]+tests/mock/mockProfile --vars " + var_str
         )
         found = executing_re.findall(captured.out)
         # We run each model separately
@@ -252,7 +252,7 @@ def test_selection(capfd):
 
 def test_no_run_results(capfd):
     with ProjectTemporaryDirectory() as tmp_dir:
-        shutil.rmtree(os.path.join(tmp_dir, "mockTarget"))
+        shutil.rmtree(os.path.join(tmp_dir, "mockTarget"), ignore_errors=True)
 
         # Without selection flag
         captured = _run_fal(