diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index c8cfbda03c..7f4d48f41e 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -942,10 +942,12 @@ def config_for_path(self, path: Path) -> t.Tuple[Config, Path]: def config_for_node(self, node: str | Model | Audit) -> Config: if isinstance(node, str): - return self.config_for_path(self.get_snapshot(node, raise_if_missing=True).node._path)[ - 0 - ] # type: ignore - return self.config_for_path(node._path)[0] # type: ignore + path = self.get_snapshot(node, raise_if_missing=True).node._path + else: + path = node._path + if path is None: + return self.config + return self.config_for_path(path)[0] # type: ignore @property def models(self) -> MappingProxyType[str, Model]: diff --git a/sqlmesh/core/macros.py b/sqlmesh/core/macros.py index 4afb5fd334..2c5ef3b2e8 100644 --- a/sqlmesh/core/macros.py +++ b/sqlmesh/core/macros.py @@ -171,7 +171,7 @@ def __init__( resolve_tables: t.Optional[t.Callable[[exp.Expression], exp.Expression]] = None, snapshots: t.Optional[t.Dict[str, Snapshot]] = None, default_catalog: t.Optional[str] = None, - path: Path = Path(), + path: t.Optional[Path] = None, environment_naming_info: t.Optional[EnvironmentNamingInfo] = None, model_fqn: t.Optional[str] = None, ): @@ -1384,7 +1384,7 @@ def normalize_macro_name(name: str) -> str: def call_macro( func: t.Callable, dialect: DialectType, - path: Path, + path: t.Optional[Path], provided_args: t.Tuple[t.Any, ...], provided_kwargs: t.Dict[str, t.Any], **optional_kwargs: t.Any, @@ -1431,7 +1431,7 @@ def _coerce( expr: t.Any, typ: t.Any, dialect: DialectType, - path: Path, + path: t.Optional[Path] = None, strict: bool = False, ) -> t.Any: """Coerces the given expression to the specified type on a best-effort basis.""" diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index 0808722119..ed93df24a7 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -1646,6 +1646,8 @@ def is_seed(self) -> bool: def seed_path(self) -> Path: seed_path = Path(self.kind.path) if not seed_path.is_absolute(): + if self._path is None: + raise SQLMeshError(f"Seed model '{self.name}' has no path") return self._path.parent / seed_path return seed_path @@ -2020,7 +2022,7 @@ def load_sql_based_model( expressions: t.List[exp.Expression], *, defaults: t.Optional[t.Dict[str, t.Any]] = None, - path: Path = Path(), + path: t.Optional[Path] = None, module_path: Path = Path(), time_column_format: str = c.DEFAULT_TIME_COLUMN_FORMAT, macros: t.Optional[MacroRegistry] = None, @@ -2171,6 +2173,8 @@ def load_sql_based_model( # The name of the model will be inferred from its path relative to `models/`, if it's not explicitly specified name = meta_fields.pop("name", "") if not name and infer_names: + if path is None: + raise ValueError(f"Model {name} must have a name") name = get_model_name(path) if not name: @@ -2249,7 +2253,7 @@ def create_seed_model( name: TableName, seed_kind: SeedKind, *, - path: Path = Path(), + path: t.Optional[Path] = None, module_path: Path = Path(), **kwargs: t.Any, ) -> Model: @@ -2268,7 +2272,12 @@ def create_seed_model( seed_path = module_path.joinpath(*subdirs) seed_kind.path = str(seed_path) elif not seed_path.is_absolute(): - seed_path = path / seed_path if path.is_dir() else path.parent / seed_path + if path is None: + seed_path = seed_path + elif path.is_dir(): + seed_path = path / seed_path + else: + seed_path = path.parent / seed_path seed = create_seed(seed_path) @@ -2403,7 +2412,7 @@ def _create_model( name: TableName, *, defaults: t.Optional[t.Dict[str, t.Any]] = None, - path: Path = Path(), + path: t.Optional[Path] = None, time_column_format: str = c.DEFAULT_TIME_COLUMN_FORMAT, jinja_macros: t.Optional[JinjaMacroRegistry] = None, jinja_macro_references: t.Optional[t.Set[MacroReference]] = None, @@ -2588,7 +2597,7 @@ def _create_model( def _split_sql_model_statements( expressions: t.List[exp.Expression], - path: Path, + path: t.Optional[Path], dialect: t.Optional[str] = None, ) -> t.Tuple[ t.Optional[exp.Expression], @@ -2709,7 +2718,7 @@ def _refs_to_sql(values: t.Any) -> exp.Expression: def render_meta_fields( fields: t.Dict[str, t.Any], module_path: Path, - path: Path, + path: t.Optional[Path], jinja_macros: t.Optional[JinjaMacroRegistry], macros: t.Optional[MacroRegistry], dialect: DialectType, @@ -2793,7 +2802,7 @@ def render_field_value(value: t.Any) -> t.Any: def render_model_defaults( defaults: t.Dict[str, t.Any], module_path: Path, - path: Path, + path: t.Optional[Path], jinja_macros: t.Optional[JinjaMacroRegistry], macros: t.Optional[MacroRegistry], dialect: DialectType, @@ -2843,7 +2852,7 @@ def parse_defaults_properties( def render_expression( expression: exp.Expression, module_path: Path, - path: Path, + path: t.Optional[Path], jinja_macros: t.Optional[JinjaMacroRegistry] = None, macros: t.Optional[MacroRegistry] = None, dialect: DialectType = None, diff --git a/sqlmesh/core/node.py b/sqlmesh/core/node.py index 874e74b3e9..4f0a66dc2e 100644 --- a/sqlmesh/core/node.py +++ b/sqlmesh/core/node.py @@ -199,7 +199,7 @@ class _Node(PydanticModel): interval_unit_: t.Optional[IntervalUnit] = Field(alias="interval_unit", default=None) tags: t.List[str] = [] stamp: t.Optional[str] = None - _path: Path = Path() + _path: t.Optional[Path] = None _data_hash: t.Optional[str] = None _metadata_hash: t.Optional[str] = None diff --git a/sqlmesh/core/renderer.py b/sqlmesh/core/renderer.py index 6622094da3..f2e9e24056 100644 --- a/sqlmesh/core/renderer.py +++ b/sqlmesh/core/renderer.py @@ -43,7 +43,7 @@ def __init__( expression: exp.Expression, dialect: DialectType, macro_definitions: t.List[d.MacroDef], - path: Path = Path(), + path: t.Optional[Path] = None, jinja_macro_registry: t.Optional[JinjaMacroRegistry] = None, python_env: t.Optional[t.Dict[str, Executable]] = None, only_execution_time: bool = False, diff --git a/sqlmesh/core/snapshot/definition.py b/sqlmesh/core/snapshot/definition.py index b6f94108a1..1a284aadfd 100644 --- a/sqlmesh/core/snapshot/definition.py +++ b/sqlmesh/core/snapshot/definition.py @@ -2198,7 +2198,7 @@ def check_ready_intervals( context: ExecutionContext, python_env: t.Dict[str, Executable], dialect: DialectType = None, - path: Path = Path(), + path: t.Optional[Path] = None, kwargs: t.Optional[t.Dict] = None, ) -> Intervals: checked_intervals: Intervals = [] diff --git a/sqlmesh/dbt/converter/convert.py b/sqlmesh/dbt/converter/convert.py index f097a83884..7eab536946 100644 --- a/sqlmesh/dbt/converter/convert.py +++ b/sqlmesh/dbt/converter/convert.py @@ -207,6 +207,8 @@ def _convert_models( if model.kind.is_seed: # this will produce the original seed file, eg "items.csv" + if model._path is None: + raise ValueError(f"Unhandled model path for model {model_name}") seed_filename = model._path.relative_to(input_paths.seeds) # seed definition - rename "items.csv" -> "items.sql" @@ -219,6 +221,8 @@ def _convert_models( assert isinstance(model.kind, SeedKind) model.kind.path = str(Path("../seeds", seed_filename)) else: + if model._path is None: + raise ValueError(f"Unhandled model path for model {model_name}") if input_paths.models in model._path.parents: model_filename = model._path.relative_to(input_paths.models) elif input_paths.snapshots in model._path.parents: @@ -290,6 +294,8 @@ def _convert_standalone_audits( audit_definition_string = ";\n".join(stringified) + if audit._path is None: + continue audit_filename = audit._path.relative_to(input_paths.tests) audit_output_path = output_paths.audits / audit_filename audit_output_path.write_text(audit_definition_string) diff --git a/sqlmesh/lsp/reference.py b/sqlmesh/lsp/reference.py index 96db4dc63d..9f1215d9ca 100644 --- a/sqlmesh/lsp/reference.py +++ b/sqlmesh/lsp/reference.py @@ -164,6 +164,9 @@ def get_model_definitions_for_a_path( else: return [] + if file_path is None: + return [] + # Find all possible references references: t.List[Reference] = [] @@ -246,6 +249,8 @@ def get_model_definitions_for_a_path( if referenced_model is None: continue referenced_model_path = referenced_model._path + if referenced_model_path is None: + continue # Check whether the path exists if not referenced_model_path.is_file(): continue @@ -372,6 +377,9 @@ def get_macro_definitions_for_a_path( else: return [] + if file_path is None: + return [] + references = [] _, config_path = lsp_context.context.config_for_path( file_path, diff --git a/sqlmesh/magics.py b/sqlmesh/magics.py index 58f7135654..ef2db145ee 100644 --- a/sqlmesh/magics.py +++ b/sqlmesh/magics.py @@ -287,8 +287,9 @@ def model(self, context: Context, line: str, sql: t.Optional[str] = None) -> Non if loaded.name == args.model: model = loaded else: - with open(model._path, "r", encoding="utf-8") as file: - expressions = parse(file.read(), default_dialect=config.dialect) + if model._path: + with open(model._path, "r", encoding="utf-8") as file: + expressions = parse(file.read(), default_dialect=config.dialect) formatted = format_model_expressions( expressions, @@ -307,8 +308,9 @@ def model(self, context: Context, line: str, sql: t.Optional[str] = None) -> Non replace=True, ) - with open(model._path, "w", encoding="utf-8") as file: - file.write(formatted) + if model._path: + with open(model._path, "w", encoding="utf-8") as file: + file.write(formatted) if sql: context.console.log_success(f"Model `{args.model}` updated") diff --git a/tests/core/test_model.py b/tests/core/test_model.py index 6fe0ccc87d..4214ce60fc 100644 --- a/tests/core/test_model.py +++ b/tests/core/test_model.py @@ -3290,7 +3290,7 @@ def runtime_macro(evaluator, **kwargs) -> None: model = load_sql_based_model(expressions) with pytest.raises( ConfigError, - match=r"Dependencies must be provided explicitly for models that can be rendered only at runtime at.*", + match=r"Dependencies must be provided explicitly for models that can be rendered only at runtime", ): model.validate_definition() @@ -8226,7 +8226,7 @@ def test_physical_version(): with pytest.raises( ConfigError, - match=r"Pinning a physical version is only supported for forward only models at.*", + match=r"Pinning a physical version is only supported for forward only models( at.*)?", ): load_sql_based_model( d.parse( diff --git a/tests/integrations/github/cicd/conftest.py b/tests/integrations/github/cicd/conftest.py index 30f65aecdc..25ba3b2d60 100644 --- a/tests/integrations/github/cicd/conftest.py +++ b/tests/integrations/github/cicd/conftest.py @@ -49,7 +49,7 @@ def _make_function(username: str, state: str, **kwargs) -> PullRequestReview: github_client.requester, {}, { - # Name is whatever they provide in their GitHub profile or login as fallback. Always use login. + # Name is whatever they provide in their GitHub profile or login as a fallback. Always use login. "user": AttributeDict(name="Unrelated", login=username), "state": state, **kwargs, diff --git a/vscode/openapi.json b/vscode/openapi.json index bf1cef0809..32a7445e32 100644 --- a/vscode/openapi.json +++ b/vscode/openapi.json @@ -1382,8 +1382,14 @@ "properties": { "name": { "type": "string", "title": "Name" }, "fqn": { "type": "string", "title": "Fqn" }, - "path": { "type": "string", "title": "Path" }, - "full_path": { "type": "string", "title": "Full Path" }, + "path": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Path" + }, + "full_path": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Full Path" + }, "dialect": { "type": "string", "title": "Dialect" }, "type": { "$ref": "#/components/schemas/ModelType" }, "columns": { @@ -1417,16 +1423,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "name", - "fqn", - "path", - "full_path", - "dialect", - "type", - "columns", - "hash" - ], + "required": ["name", "fqn", "dialect", "type", "columns", "hash"], "title": "Model" }, "ModelDetails": { @@ -2143,7 +2140,7 @@ "TestCase": { "properties": { "name": { "type": "string", "title": "Name" }, - "path": { "type": "string", "format": "path", "title": "Path" } + "path": { "type": "string", "title": "Path" } }, "additionalProperties": false, "type": "object", @@ -2153,7 +2150,7 @@ "TestErrorOrFailure": { "properties": { "name": { "type": "string", "title": "Name" }, - "path": { "type": "string", "format": "path", "title": "Path" }, + "path": { "type": "string", "title": "Path" }, "tb": { "type": "string", "title": "Tb" } }, "additionalProperties": false, @@ -2193,7 +2190,7 @@ "TestSkipped": { "properties": { "name": { "type": "string", "title": "Name" }, - "path": { "type": "string", "format": "path", "title": "Path" }, + "path": { "type": "string", "title": "Path" }, "reason": { "type": "string", "title": "Reason" } }, "additionalProperties": false, diff --git a/vscode/react/src/api/client.ts b/vscode/react/src/api/client.ts index 807f51e1d9..028b2d1912 100644 --- a/vscode/react/src/api/client.ts +++ b/vscode/react/src/api/client.ts @@ -289,6 +289,10 @@ export interface Meta { has_running_task?: boolean } +export type ModelPath = string | null + +export type ModelFullPath = string | null + export type ModelDescription = string | null export type ModelDetailsProperty = ModelDetails | null @@ -302,8 +306,8 @@ export type ModelDefaultCatalog = string | null export interface Model { name: string fqn: string - path: string - full_path: string + path?: ModelPath + full_path?: ModelFullPath dialect: string type: ModelType columns: Column[] diff --git a/vscode/react/src/pages/lineage.tsx b/vscode/react/src/pages/lineage.tsx index ddfac87fc4..18925f28da 100644 --- a/vscode/react/src/pages/lineage.tsx +++ b/vscode/react/src/pages/lineage.tsx @@ -100,9 +100,12 @@ function Lineage() { // @ts-ignore const fileUri: string = activeFile.fileUri const filePath = URI.file(fileUri).path - const model = models.find( - (m: Model) => URI.file(m.full_path).path === filePath, - ) + const model = models.find((m: Model) => { + if (!m.full_path) { + return false + } + return URI.file(m.full_path).path === filePath + }) if (model) { return model.name } @@ -195,6 +198,9 @@ export function LineageComponentFromWeb({ if (!model) { throw new Error('Model not found') } + if (!model.full_path) { + return + } vscode('openFile', { uri: URI.file(model.full_path).toString() }) } diff --git a/web/server/api/endpoints/models.py b/web/server/api/endpoints/models.py index 5d81306aea..21a7b93eb0 100644 --- a/web/server/api/endpoints/models.py +++ b/web/server/api/endpoints/models.py @@ -121,11 +121,12 @@ def serialize_model(context: Context, model: Model, render_query: bool = False) ) sql = query.sql(pretty=True, dialect=model.dialect) + path = model._path return models.Model( name=model.name, fqn=model.fqn, - path=str(model._path.absolute().relative_to(context.path).as_posix()), - full_path=str(model._path.absolute().as_posix()), + path=str(path.absolute().relative_to(context.path).as_posix()) if path else None, + full_path=str(path.absolute().as_posix()) if path else None, dialect=dialect, columns=columns, details=details, diff --git a/web/server/models.py b/web/server/models.py index cc12d036fc..d193fa5f07 100644 --- a/web/server/models.py +++ b/web/server/models.py @@ -171,8 +171,8 @@ class Column(PydanticModel): class Model(PydanticModel): name: str fqn: str - path: str - full_path: str + path: t.Optional[str] = None + full_path: t.Optional[str] = None """ As opposed to path, which is relative to the project root, full_path is the absolute path to the model file. """ diff --git a/web/server/settings.py b/web/server/settings.py index bf64504565..a96d369de8 100644 --- a/web/server/settings.py +++ b/web/server/settings.py @@ -80,7 +80,7 @@ def _get_loaded_context(path: str | Path, config: str, gateway: str) -> Context: @lru_cache() def _get_path_to_model_mapping(context: Context) -> dict[Path, Model]: - return {model._path: model for model in context._models.values()} + return {model._path: model for model in context._models.values() if model._path} def get_path_to_model_mapping(