Skip to content

Commit e5d7524

Browse files
joschajoscha
committed
fix: clean up
1 parent 58a8a3e commit e5d7524

File tree

4 files changed

+2
-157
lines changed

4 files changed

+2
-157
lines changed

‎dlt_source_notion/__init__.py

Lines changed: 0 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -9,44 +9,6 @@
99
from notion_client.helpers import iterate_paginated_api
1010

1111

12-
# logging.basicConfig(level=logging.DEBUG)
13-
14-
# if is_logging():
15-
# logger = logging.getLogger("dlt")
16-
17-
# class HideSinglePagingNonsense(logging.Filter):
18-
# def filter(self, record):
19-
# msg = record.getMessage()
20-
# if (
21-
# "Extracted data of type list from path _data with length 1" in msg
22-
# or re.match(
23-
# r"Paginator JSONLinkPaginator at [a-fA-F0-9]+: next_url_path: _meta\.links\.next\.href does not have more pages",
24-
# msg,
25-
# )
26-
# ):
27-
# return False
28-
# return True
29-
30-
# logger.addFilter(HideSinglePagingNonsense())
31-
32-
33-
# def anyurl_encoder(obj: Any) -> JsonSerializable:
34-
# if isinstance(obj, AnyUrl):
35-
# return obj.unicode_string()
36-
# raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
37-
38-
39-
# json.set_custom_encoder(anyurl_encoder)
40-
41-
42-
# def pydantic_model_dump(model: BaseModel, **kwargs):
43-
# """
44-
# Dumps a Pydantic model to a dictionary, using the model's field names as keys and NOT observing the field aliases,
45-
# which is important for DLT to correctly map the data to the destination.
46-
# """
47-
# return model.model_dump(by_alias=True, **kwargs)
48-
49-
5012
class Table(StrEnum):
5113
PERSONS = "persons"
5214
BOTS = "bots"
@@ -56,72 +18,12 @@ def use_id(entity: Dict, **kwargs) -> Dict:
5618
return filter_dict(entity, **kwargs) | {"_dlt_id": __get_id(entity)}
5719

5820

59-
# @dlt.resource(
60-
# selected=True,
61-
# parallelized=True,
62-
# primary_key="id",
63-
# )
64-
# def persons(rest_client: RESTClient) -> Iterable[TDataItem]:
65-
# for persons_raw in rest_client.paginate(
66-
# V2_PERSONS, params={"limit": V2_MAX_PAGE_LIMIT}, hooks=hooks
67-
# ):
68-
# yield persons_adapter.validate_python(persons_raw)
69-
70-
71-
# async def person_employments(
72-
# person: Person,
73-
# rest_client: RESTClient,
74-
# ):
75-
# href = jmespath.search("links.employments.href", person.field_meta)
76-
# if not href:
77-
# return
78-
# for employments_raw in rest_client.paginate(
79-
# href, params={"limit": V2_MAX_PAGE_LIMIT}, hooks=hooks
80-
# ):
81-
# employments = employments_adapter.validate_python(employments_raw)
82-
# for employment in employments:
83-
# yield dlt.mark.with_hints(
84-
# item=use_id(employment, exclude=["field_meta", "org_units"]),
85-
# hints=dlt.mark.make_hints(
86-
# table_name=Table.EMPLOYMENTS.value,
87-
# ),
88-
# # needs to be a variant due to https://github.com/dlt-hub/dlt/pull/2109
89-
# create_table_variant=True,
90-
# )
91-
92-
9321
def __get_id(obj):
9422
if isinstance(obj, dict):
9523
return obj.get("id")
9624
return getattr(obj, "id", None)
9725

9826

99-
# @dlt.transformer(
100-
# max_table_nesting=1,
101-
# parallelized=True,
102-
# table_name=Table.PERSONS.value,
103-
# )
104-
# async def person_details(persons: List[Person], rest_client: RESTClient):
105-
# yield [
106-
# use_id(person, exclude=["field_meta", "custom_attributes", "employments"])
107-
# for person in persons
108-
# ]
109-
# for person in persons:
110-
# yield person_employments(person, rest_client)
111-
# yield dlt.mark.with_hints(
112-
# item={"person_id": person.id}
113-
# | {cas.root.id: cas.root.value for cas in person.custom_attributes},
114-
# hints=dlt.mark.make_hints(
115-
# table_name=Table.CUSTOM_ATTRIBUTES.value,
116-
# primary_key="person_id",
117-
# merge_key="person_id",
118-
# write_disposition="merge",
119-
# ),
120-
# # needs to be a variant due to https://github.com/dlt-hub/dlt/pull/2109
121-
# create_table_variant=True,
122-
# )
123-
124-
12527
@dlt.resource(
12628
selected=True,
12729
parallelized=True,

‎dlt_source_notion/client.py

Lines changed: 0 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -4,60 +4,3 @@
44

55
def get_notion_client():
66
return Client(auth=dlt.secrets["notion_token"])
7-
8-
9-
# Share a session (and thus pool) between all rest clients
10-
# session: Session = Session(raise_for_status=False)
11-
12-
# auth: notionOAuth2ClientCredentials = None
13-
14-
15-
# def get_rest_client(
16-
# api_base: str = API_BASE,
17-
# ):
18-
# global session
19-
# global auth
20-
21-
# if auth is None:
22-
# auth = notionOAuth2ClientCredentials(
23-
# api_base=api_base,
24-
# client_id=dlt.secrets["notion_client_id"],
25-
# client_secret=dlt.secrets["notion_client_secret"],
26-
# default_token_expiration=86400,
27-
# session=session,
28-
# )
29-
30-
# client = RESTClient(
31-
# base_url=api_base,
32-
# headers={
33-
# "Accept": "application/json",
34-
# "X-notion-App-ID": X_notion_APP_ID,
35-
# },
36-
# auth=auth,
37-
# data_selector="_data",
38-
# paginator=JSONLinkPaginator(next_url_path="_meta.links.next.href"),
39-
# session=session,
40-
# )
41-
# return client, auth
42-
43-
44-
# def debug_response(response: Response, *args: Any, **kwargs: Any) -> None:
45-
# if logging.getLogger().isEnabledFor(logging.DEBUG):
46-
# logging.debug(
47-
# f"Response: {response.status_code} {response.reason} {response.url} {response.text}"
48-
# )
49-
50-
51-
# def raise_for_status(response: Response, *args: Any, **kwargs: Any) -> None:
52-
# # TODO: Add more detailed error handling, using the notion error response model
53-
# response.raise_for_status()
54-
55-
56-
# hooks = {
57-
# "response": [
58-
# debug_response,
59-
# raise_for_status,
60-
# ]
61-
# }
62-
# V2_MAX_PAGE_LIMIT = 50
63-
# V1_BASE_PAGE = 0

‎dlt_source_notion/tests/__init__.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +0,0 @@
1-
def test_anything():
2-
pass

‎dlt_source_notion/tests/test_noop.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
def test_noop():
2+
pass

0 commit comments

Comments
 (0)