Update generator for new lemmy API

type-generator/headers/object_header.py

@@ -1,3 +1,2 @@
 from dataclasses import dataclass
-from typing import Optional, Any
-
+from typing import Optional, Any

type-generator/headers/response_header.py

@@ -4,6 +4,4 @@
 
 from .views import *
 from .objects import *
-from .types import ResponseWrapper
-
-
+from .types import ResponseWrapper

type-generator/headers/view_header.py

@@ -1,6 +1,4 @@
 from dataclasses import dataclass
 from typing import Optional, Any
 
-from .objects import *
-
-
+from .objects import *

type-generator/main.py

@@ -7,31 +7,58 @@
 from openapi_parser.parser import OpenApiParser
 from tree_sitter import Parser, Language
 import tree_sitter_typescript as ts_typescript
-from src import ModelVisitor, EnumVisitor, HttpVisitor, ClassType, ModelGenerator, HttpGenerator, Property
+from src import ModelVisitor, EnumVisitor, HttpVisitor, ClassType, ModelGenerator, HttpGenerator, Property, \
+    TypeAliasVisitor
 
 parser = Parser()
 parser.set_language(Language(ts_typescript.language_typescript(), "TypeScript"))
 model_dir = "../pythonlemmy"
 # model_dir = "./test_output"
 
 enum_names = []
+type_aliases = {}
 
-objects = []
-responses = []
-views = []
+objects = {}
+object_dependencies = {}
+responses = {}
+response_dependencies = {}
+views = {}
+view_dependencies = {}
 
 openapi_docs = "https://raw.githubusercontent.com/MV-GH/lemmy_openapi_spec/master/lemmy_spec.yaml"
 
 
+# From https://code.activestate.com/recipes/576570-dependency-resolver/
+def dep(arg):
+    '''
+        Dependency resolver
+
+    "arg" is a dependency dictionary in which
+    the values are the dependencies of their respective keys.
+    '''
+    d=dict((k, set(arg[k])) for k in arg)
+    r=[]
+    while d:
+        # values not in keys (items without dep)
+        t=set(i for v in d.values() for i in v)-set(d.keys())
+        # and keys without value (items without dep)
+        t.update(k for k, v in d.items() if not v)
+        # can be done right away
+        r.append(t)
+        # and cleaned up
+        d=dict(((k, v-t) for k, v in d.items() if v))
+    return r
+
+
 def list_enums():
     types_dir = f"{current_dir()}lemmy-js-client/src/types/"
     files = os.listdir(types_dir)
 
     for file in files:
-        if file.endswith("Id.ts"):
-            continue
         with open(f"{types_dir}{file}", "r") as f:
-            parse_enum(f.read())
+            content = f.read()
+        parse_enum(content)
+        parse_type_alias(content)
 
 
 def generate_types():
@@ -42,29 +69,31 @@ def generate_types():
         os.makedirs(model_dir)
 
     for file in files:
-        if file.endswith("Id.ts"):
+        file_without_extension = file[:-len(".ts")]
+        if file_without_extension in enum_names or file_without_extension in type_aliases.keys():
             continue
         with open(f"{types_dir}{file}", "r") as f:
             parse_model(f.read())
 
-    with open(f"./headers/object_header.py", "r") as f:
-        object_header = f.read()
-    with open(f"./headers/response_header.py", "r") as f:
-        response_header = f.read()
-    with open(f"./headers/view_header.py", "r") as f:
-        view_header = f.read()
+    print(f"Object count = {len(objects)}, Response count = {len(responses)}, View count = {len(views)}")
+    generate_file(f"./headers/object_header.py", "objects.py", objects, object_dependencies)
+    generate_file(f"./headers/response_header.py", "responses.py", responses, response_dependencies)
+    generate_file(f"./headers/view_header.py", "views.py", views, view_dependencies)
 
-    with open(f"{model_dir}/views.py", "w") as f:
-        f.write(view_header)
-        f.write("\n\n\n".join(views))
-        f.write("\n")
-    with open(f"{model_dir}/objects.py", "w") as f:
-        f.write(object_header)
-        f.write("\n\n\n".join(objects))
-        f.write("\n")
-    with open(f"{model_dir}/responses.py", "w") as f:
-        f.write(response_header)
-        f.write("\n\n\n".join(responses))
+
+def generate_file(header_file_path: str, output_file_path: str, types: dict[str, str], dependencies: dict[str, str]):
+    with open(header_file_path, "r") as f:
+        header = f.read()
+
+    dep_tree = dep(dependencies)
+
+    with open(f"{model_dir}/{output_file_path}", "w") as f:
+        f.write(header)
+        for d in dep_tree:
+            for r in d:
+                if r not in types:
+                    continue
+                f.write(f"\n\n\n{types[r]}")
         f.write("\n")
 
 
@@ -77,23 +106,45 @@ def parse_enum(model_contents: str):
     visitor = EnumVisitor(tree)
     visitor.walk()
 
+    if not visitor.is_enum:
+        return
+
     enum_names.append(visitor.enum_name)
 
 
+def parse_type_alias(model_contents: str):
+    tree = parser.parse(bytes(model_contents, "utf-8"))
+
+    if "export type" not in model_contents:
+        return
+
+    visitor = TypeAliasVisitor(tree)
+    visitor.walk()
+
+    if not visitor.is_type_alias:
+        return
+
+    type_aliases[visitor.type_alias_name] = visitor.mapped_type
+
+
 def parse_model(model_contents: str):
     tree = parser.parse(bytes(model_contents, "utf-8"))
 
-    if "export interface" not in model_contents:
+    if "export type" not in model_contents:
         return
-    visitor = ModelVisitor(tree, enum_names)
+
+    visitor = ModelVisitor(tree, enum_names, type_aliases)
     visitor.walk()
     result = ModelGenerator(visitor.class_name, visitor.properties, visitor.class_type).build()
     if visitor.class_type == ClassType.VIEW:
-        views.append(result)
+        views[visitor.class_name] = result
+        view_dependencies[visitor.class_name] = visitor.dependencies
     elif visitor.class_type == ClassType.RESPONSE:
-        responses.append(result)
+        responses[visitor.class_name] = result
+        response_dependencies[visitor.class_name] = visitor.dependencies
     elif visitor.class_type == ClassType.OBJECT:
-        objects.append(result)
+        objects[visitor.class_name] = result
+        object_dependencies[visitor.class_name] = visitor.dependencies
 
 
 def generate_http():
@@ -117,7 +168,7 @@ def parse_http() -> str:
         tree = parser.parse(bytes(f.read(), "utf-8"))
         visitor = HttpVisitor(tree)
         visitor.walk()
-        result = HttpGenerator(visitor.methods, types_dir, enum_names, docs).build()
+        result = HttpGenerator(visitor.methods, types_dir, enum_names, type_aliases, docs).build()
         return result
 
 

type-generator/src/generator/http_generator.py

@@ -1,5 +1,5 @@
 import textwrap
-from typing import List, Optional
+from typing import List, Optional, Dict
 
 from openapi_parser.model import ModelEndpoint, ModelSchema, ModelEnumData
 from openapi_parser.parser import OpenApiParser, ModelEnumDataImpl
@@ -32,10 +32,11 @@ class HttpGenerator:
 
     _methods: List[ApiMethod] = []
 
-    def __init__(self, methods: List[ApiMethod], types_dir: str, enums: List[str],
+    def __init__(self, methods: List[ApiMethod], types_dir: str, enums: List[str], type_aliases: Dict[str, str],
                  openapi: Optional[OpenApiParser] = None):
         self._methods = methods
         self._types_dir = types_dir
+        self._type_aliases = type_aliases
         self._enums = enums
         self._openapi = openapi
 
@@ -147,6 +148,6 @@ def _get_properties(self, method: ApiMethod) -> List[Property]:
 
         with open(f"{self._types_dir}/{method.input}.ts", "r") as f:
             tree = parser.parse(bytes(f.read(), "utf-8"))
-            visitor = ModelVisitor(tree, self._enums)
+            visitor = ModelVisitor(tree, self._enums, self._type_aliases)
         visitor.walk()
         return visitor.properties

type-generator/src/visitor/__init__.py

@@ -1,3 +1,4 @@
 from .model_visitor import ModelVisitor
 from .enum_visitor import EnumVisitor
 from .http_visitor import HttpVisitor
+from .type_alias_visitor import TypeAliasVisitor

type-generator/src/visitor/enum_visitor.py

@@ -11,14 +11,20 @@ class EnumVisitor(Visitor):
     _encoding = "utf-8"
     _number_type = "long"
 
+    is_enum: bool
     enum_name = ""
     types: List[EnumProperty] = []
 
     def __init__(self, tree: Tree):
         self.tree = tree
+        self.is_enum = False
         self.enum_name = ""
         self.types = []
 
+    def visit_union_type(self, node: Node):
+        self.is_enum = True
+        self._accept_list(node.children)
+
     def visit_type_alias_declaration(self, node: Node):
         self.enum_name = node.child_by_field_name("name").text.decode(self._encoding)
         self._accept_list(node.children)

type-generator/src/visitor/model_visitor.py

@@ -1,5 +1,5 @@
 import textwrap
-from typing import List
+from typing import List, Dict
 
 from tree_sitter import Node, Tree
 
@@ -15,15 +15,18 @@ class ModelVisitor(Visitor):
     class_name = ""
     class_type = ClassType.OBJECT
     properties: List[Property] = []
+    dependencies: List[str] = []
 
-    def __init__(self, tree: Tree, enums: List[str]):
+    def __init__(self, tree: Tree, enums: List[str], type_aliases: Dict[str, str]):
         self.tree = tree
         self._enums = enums
         self.class_name = ""
         self.class_type = ClassType.OBJECT
         self.properties = []
+        self._type_aliases = type_aliases
+        self.dependencies = []
 
-    def visit_interface_declaration(self, node: Node):
+    def visit_type_alias_declaration(self, node: Node):
         self.class_name = node.child_by_field_name("name").text.decode(self._encoding)
 
         if self.class_name.endswith("Response"):
@@ -50,19 +53,27 @@ def visit_type_annotation(self, node: Node):
         type_identifier = node.child(1).text.decode(self._encoding)
         if "<" in type_identifier:
             return self._accept(node.child(1))
+        if type_identifier == "null":
+            return self._handle_null()
 
         if type_identifier in self._enums:
             type_identifier = "str"
+        elif type_identifier in self._type_aliases.keys():
+            type_identifier = self._type_aliases[type_identifier]
 
         last_idx = len(self.properties) - 1
 
-        self.properties[last_idx].type = normalize_type(type_identifier)
+        type_identifier = normalize_type(type_identifier)
+        self.properties[last_idx].type = type_identifier
+        self._update_dependency_tree(type_identifier)
 
     def visit_generic_type(self, node: Node):
         last_idx = len(self.properties) - 1
         type_identifier = node.child(0).text.decode(self._encoding)
         type_parameter = normalize_type(node.child(1).child(1).text.decode(self._encoding))
 
+        if type_parameter == "null":
+            return self._handle_null()
         if type_parameter in self._enums:
             type_parameter = "str"
 
@@ -72,3 +83,12 @@ def visit_generic_type(self, node: Node):
             raise f"Unhandled Type {type_identifier}!"
 
         self.properties[last_idx].type = out_type
+        self._update_dependency_tree(type_parameter)
+
+    def _handle_null(self):
+        self.properties.pop()
+
+    def _update_dependency_tree(self, type_name: str):
+        if type_name in ["str", "int", "bool"]:
+            return
+        self.dependencies.append(type_name)

type-generator/src/visitor/type_alias_visitor.py

@@ -0,0 +1,31 @@
+import textwrap
+from typing import List
+
+from tree_sitter import Node, Tree
+from .visitor import Visitor
+from ..models import EnumProperty
+from ..util import to_enum_case
+
+
+class TypeAliasVisitor(Visitor):
+    _encoding = "utf-8"
+    _number_type = "long"
+
+    is_type_alias: bool
+    type_alias_name = ""
+    mapped_type = ""
+
+    def __init__(self, tree: Tree):
+        self.tree = tree
+        self.is_type_alias = False
+        self.type_alias_name = ""
+        self.mapped_type = ""
+
+    def visit_type_alias_declaration(self, node: Node):
+        self.type_alias_name = node.child_by_field_name("name").text.decode(self._encoding)
+        if node.child_by_field_name("value").type == "predefined_type":
+            self.is_type_alias = True
+            self._accept_list(node.children)
+
+    def visit_predefined_type(self, node: Node):
+        self.mapped_type = node.text.decode(self._encoding)