Generator fails for missing project_dir #126
Description
I'm encountering some blocking issues when trying to generate the API source for posthog.
So just running the cli with no configuration whatsoever I get this output:
$ dlt-init-openapi --url https://app.posthog.com/api/schema/
2024-07-02 18:13:50.181 | SUCCESS | dlt_init_openapi.cli:_init_command_wrapped:91 - Starting dlt openapi generator
2024-07-02 18:13:50.181 | INFO | dlt_init_openapi.utils.update_rest_api:update_rest_api:14 - Syncing rest_api verified source
2024-07-02 18:13:50.182 | INFO | dlt_init_openapi.utils.update_rest_api:update_rest_api:18 - rest_api verified source already present
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /home/drnick/repos/analytics/42next-modeling/.venv/lib/python3.10/site-packages/dlt_init_openapi │
│ /cli/__init__.py:58 in init │
│ │
│ 55 ) -> None: │
│ 56 │ """Generate a new dlt pipeline""" │
│ 57 │ │
│ ❱ 58 │ _init_command_wrapped( │
│ 59 │ │ source=source, │
│ 60 │ │ url=url, │
│ 61 │ │ path=path, │
│ │
│ ╭──────────────────────────── locals ────────────────────────────╮ │
│ │ allow_openapi_2 = False │ │
│ │ config_path = None │ │
│ │ global_limit = 0 │ │
│ │ interactive = True │ │
│ │ log_level = 20 │ │
│ │ output_path = None │ │
│ │ path = None │ │
│ │ source = None │ │
│ │ update_rest_api_source = False │ │
│ │ url = 'https://app.posthog.com/api/schema/' │ │
│ │ version = None │ │
│ ╰────────────────────────────────────────────────────────────────╯ │
│ │
│ /home/drnick/repos/analytics/42next-modeling/.venv/lib/python3.10/site-packages/dlt/common/runti │
│ me/telemetry.py:97 in _wrap │
│ │
│ 94 │ │ │ │ return f(*f_args, **f_kwargs) │
│ 95 │ │ │ # some commands we track after, where we can pass the success │
│ 96 │ │ │ try: │
│ ❱ 97 │ │ │ │ rv = f(*f_args, **f_kwargs) │
│ 98 │ │ │ │ # if decorated function returns int, 0 is a success - used to track dlt │
│ 99 │ │ │ │ if isinstance(rv, int): │
│ 100 │ │ │ │ │ success = rv == 0 │
│ │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │ _track = <function with_telemetry.<locals>.decorator.<locals>._wrap.<locals>._track at │ │
│ │ 0x7fe9796e1f30> │ │
│ │ args = ('source', 'url', 'path') │ │
│ │ bound_args = <BoundArguments (source=None, url='https://app.posthog.com/api/schema/', │ │
│ │ path=None, output_path=None, config_path=None, interactive=True, │ │
│ │ log_level=20, global_limit=0, update_rest_api_source=False, │ │
│ │ allow_openapi_2=False)> │ │
│ │ category = 'command' │ │
│ │ command = 'init-openapi' │ │
│ │ f = <function _init_command_wrapped at 0x7fe9796e1e10> │ │
│ │ f_args = () │ │
│ │ f_kwargs = { │ │
│ │ │ 'source': None, │ │
│ │ │ 'url': 'https://app.posthog.com/api/schema/', │ │
│ │ │ 'path': None, │ │
│ │ │ 'output_path': None, │ │
│ │ │ 'config_path': None, │ │
│ │ │ 'interactive': True, │ │
│ │ │ 'log_level': 20, │ │
│ │ │ 'global_limit': 0, │ │
│ │ │ 'update_rest_api_source': False, │ │
│ │ │ 'allow_openapi_2': False │ │
│ │ } │ │
│ │ props = { │ │
│ │ │ 'source': None, │ │
│ │ │ 'url': 'https://app.posthog.com/api/schema/', │ │
│ │ │ 'path': None, │ │
│ │ │ 'elapsed': 0.003656625747680664, │ │
│ │ │ 'success': False, │ │
│ │ │ 'event_category': 'command', │ │
│ │ │ 'event_name': 'init-openapi' │ │
│ │ } │ │
│ │ sig = <Signature (source: str, url: Optional[str] = None, path: │ │
│ │ Optional[pathlib.Path] = None, output_path: Optional[pathlib.Path] = None, │ │
│ │ config_path: Optional[pathlib.Path] = None, interactive: bool = True, │ │
│ │ log_level: int = 20, global_limit: int = 0, update_rest_api_source: bool = │ │
│ │ False, allow_openapi_2: bool = False) -> None> │ │
│ │ start_ts = 1719936830.178589 │ │
│ │ track_before = False │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
│ │
│ /home/drnick/repos/analytics/42next-modeling/.venv/lib/python3.10/site-packages/dlt_init_openapi │
│ /cli/__init__.py:119 in _init_command_wrapped │
│ │
│ 116 │ │ │ }, │
│ 117 │ │ ) │
│ 118 │ │ │
│ ❱ 119 │ │ if config.project_dir.exists(): │
│ 120 │ │ │ if not interactive: │
│ 121 │ │ │ │ logger.info("Non interactive mode selected, overwriting existing source. │
│ 122 │ │ │ elif not questionary.confirm( │
│ │
│ ╭─────────────────────────────────────────── locals ───────────────────────────────────────────╮ │
│ │ allow_openapi_2 = False │ │
│ │ config = Config( │ │
│ │ │ output_path=None, │ │
│ │ │ project_name=None, │ │
│ │ │ package_name=None, │ │
│ │ │ post_hooks=[ │ │
│ │ │ │ 'autoflake -i -r --remove-all-unused-imports │ │
│ │ --remove-unused-variables .', │ │
│ │ │ │ 'isort --float-to-top .', │ │
│ │ │ │ 'black .' │ │
│ │ │ ], │ │
│ │ │ include_methods=['get'], │ │
│ │ │ fallback_openapi_title='openapi', │ │
│ │ │ project_folder_suffix='_pipeline', │ │
│ │ │ pipeline_file_suffix='_pipeline.py', │ │
│ │ │ dataset_name_suffix='_data', │ │
│ │ │ endpoint_filter=<function questionary_endpoint_selection at │ │
│ │ 0x7fe9796e1990>, │ │
│ │ │ name_resources_by_operation=False, │ │
│ │ │ │ │
│ │ renderer_class='dlt_init_openapi.renderer.default.DefaultRenderer', │ │
│ │ │ │ │
│ │ detector_class='dlt_init_openapi.detector.default.DefaultDetector', │ │
│ │ │ global_limit=0, │ │
│ │ │ required_parameter_default_value='FILL_ME_IN', │ │
│ │ │ unrequired_parameter_default_value='OPTIONAL_CONFIG', │ │
│ │ │ allow_openapi_2=False, │ │
│ │ │ project_dir=None, │ │
│ │ │ pipeline_file_name=None, │ │
│ │ │ spec_url='https://app.posthog.com/api/schema/', │ │
│ │ │ spec_path=None │ │
│ │ ) │ │
│ │ config_path = None │ │
│ │ create_new_client = <function create_new_client at 0x7fe979a69750> │ │
│ │ global_limit = 0 │ │
│ │ interactive = True │ │
│ │ log_level = 20 │ │
│ │ output_path = None │ │
│ │ path = None │ │
│ │ source = None │ │
│ │ update_rest_api_source = False │ │
│ │ url = 'https://app.posthog.com/api/schema/' │ │
│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
AttributeError: 'NoneType' object has no attribute 'exists'So it seems that config.project_dir is None. Seeing as there is no way to configure it without passing a config.yaml, I then created one which is very simple
project_name: posthog
project_folder_suffix: testIt fails with an cryptic error
$ dlt-init-openapi --url https://app.posthog.com/api/schema/ --config config.yaml
2024-07-02 18:37:39.111 | SUCCESS | dlt_init_openapi.cli:_init_command_wrapped:91 - Starting dlt openapi generator
2024-07-02 18:37:39.112 | INFO | dlt_init_openapi.utils.update_rest_api:update_rest_api:14 - Syncing rest_api verified source
2024-07-02 18:37:39.112 | INFO | dlt_init_openapi.utils.update_rest_api:update_rest_api:18 - rest_api verified source already present
Usage: dlt-init-openapi [OPTIONS] [SOURCE]
Try 'dlt-init-openapi --help' for help.
╭─ Error ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
│ Invalid value: Unable to parse config │
╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯So then I was debugging the cli and it turns out that the real error is here
def load_from_path(path: Path, *args: Any, **kwargs: Any) -> "Config":
"""Creates a Config from provided JSON or YAML file and sets a bunch of globals from it"""
mime = mimetypes.guess_type(path.absolute().as_uri(), strict=True)[0]
if mime == "application/json":
config_data = json.loads(path.read_text())
else:
config_data = yaml.safe_load(path.read_text())
config = Config(**config_data, **kwargs) ## This is the real error
return configTypeError("dlt_init_openapi.config.Config() got multiple values for keyword argument 'project_name'")
This is because the keyword is set to None in config_data and in the kwargs
config_data = {'project_folder_suffix': 'test', 'project_name': 'posthog'}
kwargs = {'project_name': None, 'package_name': None, 'output_path': None, 'endpoint_filter': <function questionary_endpoint_selection at 0x7a85485e9750>, 'global_limit': 0, 'spec_url': 'https://app.posthog.com/api/schema/', 'spec_path': None, 'allow_openapi_2': False}and I can't seem to find a workaround without rewriting the cli. Pydantic (or is it python) doesn't support multiple keywords.