Merge branch 'main' into cache-negative-lookups

Author: tetron

.flake8

@@ -1,7 +1,10 @@
 [flake8]
-ignore = E203, E266, E501, W503, E211, E731
-max-line-length = 88
-select = B,C,E,F,W,T4,B9
+ignore = E203,W503
+max-line-length = 100
+select = B,C,E,F,W,T4
+extend-ignore = E501,B905
+# when Python 3.10 is the minimum version, re-enable check B905 for zip + strict
+extend-select = B9
 per-file-ignores =
   schema_salad/metaschema.py:B950
   schema_salad/tests/*.py:B011

Makefile

@@ -220,8 +220,11 @@ release:
 		twine upload testenv2/src/${PACKAGE}/dist/* && \
 		git tag ${VERSION} && git push --tags
 
-flake8: $(PYSOURCES)
-	flake8 $^
+flake8: FORCE
+	flake8 $(PYSOURCES)
+
+schema_salad/metaschema.py: schema_salad/codegen_base.py schema_salad/python_codegen_support.py schema_salad/python_codegen.py schema_salad/metaschema
+	schema-salad-tool --codegen python schema_salad/metaschema/metaschema.yml > $@
 
 FORCE:
 

dev-requirements.txt

@@ -1,7 +1,8 @@
 diff_cover < 7.5
 pylint < 2.16
 pydocstyle < 6.4
-flake8-bugbear < 22.13
+flake8 > 4
+flake8-bugbear < 23.2
 tox < 3.29  # until tox-pyenv is updated for tox 4.x
 tox-pyenv < 1.2
 isort < 5.12

schema_salad/__init__.py

@@ -1,4 +1,4 @@
-"""Salad is a schema language for describing JSON or YAML structured linked data documents."""
+"""A schema language for describing JSON or YAML structured linked data documents."""
 
 import logging
 import sys

schema_salad/avro/schema.py

@@ -605,9 +605,8 @@ def make_avsc_object(json_data: JsonDataType, names: Optional[Names] = None) ->
                 )
             if not (doc is None or isinstance(doc, str) or isinstance(doc, list)):
                 raise SchemaParseException(
-                    '"doc" for type {} must be a string, a list of strings, or None: {}'.format(
-                        atype, json_data
-                    )
+                    f'"doc" for type {atype} must be a string, '
+                    f"a list of strings, or None: {json_data}"
                 )
             if atype == "enum":
                 symbols = json_data.get("symbols")

schema_salad/codegen.py

@@ -43,7 +43,7 @@ def codegen(
     """Generate classes with loaders for the given Schema Salad description."""
     j = schema.extend_and_specialize(i, loader)
 
-    gen = None  # type: Optional[CodeGenBase]
+    gen: Optional[CodeGenBase] = None
     base = schema_metadata.get("$base", schema_metadata.get("id"))
     # ``urlsplit`` decides whether to return an encoded result based
     # on the object type. To ensure the code behaves the same for Py

schema_salad/cpp_codegen.py

@@ -7,23 +7,20 @@
 The generated code requires the libyaml-cpp library & headers
 
 To see an example of usage, look at schema_salad/tests/codegen/cwl.cpp
-which can be combined with the CWL V1.0 schema as shown below:
+which can be combined with the CWL V1.0 schema as shown below::
 
-```
-schema-salad-tool --codegen cpp schema_salad/tests/test_schema/CommonWorkflowLanguage.yml \
-        > cwl_v1_0.h
+  schema-salad-tool --codegen cpp \
+          schema_salad/tests/test_schema/CommonWorkflowLanguage.yml \
+          > cwl_v1_0.h
 
-g++ --std=c++20 -I. -lyaml-cpp schema_salad/tests/codegen/cwl.cpp -o cwl-v1_0-test
-./cwl-v1_0-test
+  g++ --std=c++20 -I. -lyaml-cpp schema_salad/tests/codegen/cwl.cpp -o cwl-v1_0-test
+  ./cwl-v1_0-test
 
-# g++ versions older than version 10 may need "--std=c++2a" instead of "--std=c++20"
-```
+  # g++ versions older than version 10 may need "--std=c++2a" instead of "--std=c++20"
 """
 import re
 from typing import IO, Any, Dict, List, Optional, Tuple, Union, cast
 
-from schema_salad.utils import aslist
-
 from . import _logger
 from .codegen_base import CodeGenBase, TypeDef
 from .exceptions import SchemaException
@@ -161,7 +158,6 @@ def writeDefinition(
         self, target: IO[Any], fullInd: str, ind: str, namespace: str
     ) -> None:
         name = safename(self.name)
-        # target.write(f"{fullInd}std::unique_ptr<{self.typeStr}> {name} = std::make_unique<{self.typeStr}>();\n")
         typeStr = self.typeStr.replace(namespace + "::", "")
         target.write(f"{fullInd}heap_object<{typeStr}> {name};\n")
 
@@ -599,8 +595,6 @@ def parseEnum(self, stype: Dict[str, Any]) -> str:
         return name
 
     def parse(self, items: List[Dict[str, Any]]) -> None:
-        types = {i["name"]: i for i in items}  # type: Dict[str, Any]
-
         for stype in items:
             if "type" in stype and stype["type"] == "documentation":
                 continue

schema_salad/exceptions.py

@@ -16,14 +16,14 @@ def __init__(
     ) -> None:
         super().__init__(msg)
         self.message = self.args[0]
-        self.file = None  # type: Optional[str]
-        self.start = None  # type: Optional[Tuple[int, int]]
-        self.end = None  # type: Optional[Tuple[int, int]]
+        self.file: Optional[str] = None
+        self.start: Optional[Tuple[int, int]] = None
+        self.end: Optional[Tuple[int, int]] = None
 
-        self.is_warning = False  # type: bool
+        self.is_warning: bool = False
 
         # It will be set by its parent
-        self.bullet = ""  # type: str
+        self.bullet: str = ""
 
         def simplify(exc: "SchemaSaladException") -> List["SchemaSaladException"]:
             return [exc] if len(exc.message) else exc.children
@@ -36,7 +36,7 @@ def with_bullet(
             return exc
 
         if children is None:
-            self.children = []  # type: List["SchemaSaladException"]
+            self.children: List["SchemaSaladException"] = []
         elif len(children) <= 1:
             self.children = sum((simplify(c) for c in children), [])
         else:
@@ -82,10 +82,10 @@ def leaves(self) -> List["SchemaSaladException"]:
         return []
 
     def prefix(self) -> str:
-        pre = ""  # type:str
+        pre: str = ""
         if self.file:
-            linecol0 = ""  # type: Union[int, str]
-            linecol1 = ""  # type: Union[int, str]
+            linecol0: Union[int, str] = ""
+            linecol1: Union[int, str] = ""
             if self.start:
                 linecol0, linecol1 = self.start
             pre = f"{self.file}:{linecol0}:{linecol1}: "

schema_salad/makedoc.py

@@ -45,7 +45,7 @@ def vocab_type_name(url: str) -> str:
 
 
 def has_types(items: Any) -> List[str]:
-    r = []  # type: List[str]
+    r: List[str] = []
     if isinstance(items, MutableMapping):
         if items["type"] == "https://w3id.org/cwl/salad#record":
             return [items["name"]]
@@ -106,8 +106,9 @@ def block_code(self, code: str, info: Optional[str] = None) -> str:
         return text + ">" + html.escape(code, quote=self._escape) + "</code></pre>\n"
 
 
-def markdown_list_hook(markdown, text, state):
-    # type: (Markdown[str, Any], str, State) -> Tuple[str, State]
+def markdown_list_hook(
+    markdown: "Markdown[str, Any]", text: str, state: "State"
+) -> Tuple[str, "State"]:
     """Patches problematic Markdown lists for later HTML generation.
 
     When a Markdown list with paragraphs not indented with the list
@@ -286,7 +287,8 @@ def __init__(self) -> None:
         self.toc = ""
         self.start_numbering = True
 
-    def add_entry(self, thisdepth, title):  # type: (int, str) -> str
+    def add_entry(self, thisdepth: int, title: str) -> str:
+        """Add an entry to the table of contents."""
         depth = len(self.numbering)
         if thisdepth < depth:
             self.toc += "</ol>"
@@ -390,12 +392,12 @@ def __init__(
     ) -> None:
         self.typedoc = StringIO()
         self.toc = toc
-        self.subs = {}  # type: Dict[str, str]
-        self.docParent = {}  # type: Dict[str, List[str]]
-        self.docAfter = {}  # type: Dict[str, List[str]]
-        self.rendered = set()  # type: Set[str]
+        self.subs: Dict[str, str] = {}
+        self.docParent: Dict[str, List[str]] = {}
+        self.docAfter: Dict[str, List[str]] = {}
+        self.rendered: Set[str] = set()
         self.redirects = redirects
-        self.title = None  # type: Optional[str]
+        self.title: Optional[str] = None
         self.primitiveType = primitiveType
 
         for t in j:
@@ -418,17 +420,15 @@ def __init__(
         metaschema_loader = get_metaschema()[2]
         alltypes = extend_and_specialize(j, metaschema_loader)
 
-        self.typemap = {}  # type: Dict[str, Dict[str, str]]
-        self.uses = {}  # type: Dict[str, List[Tuple[str, str]]]
-        self.record_refs = {}  # type: Dict[str, List[str]]
+        self.typemap: Dict[str, Dict[str, str]] = {}
+        self.uses: Dict[str, List[Tuple[str, str]]] = {}
+        self.record_refs: Dict[str, List[str]] = {}
         for entry in alltypes:
             self.typemap[entry["name"]] = entry
             try:
                 if entry["type"] == "record":
                     self.record_refs[entry["name"]] = []
-                    fields = entry.get(
-                        "fields", []
-                    )  # type: Union[str, List[Dict[str, str]]]
+                    fields: Union[str, List[Dict[str, str]]] = entry.get("fields", [])
                     if isinstance(fields, str):
                         raise KeyError("record fields must be a list of mappings")
                     for f in fields:  # type: Dict[str, str]
@@ -615,20 +615,20 @@ def extendsfrom(item: Dict[str, Any], ex: List[Dict[str, Any]]) -> None:
             f["doc"] = number_headings(self.toc, f["doc"])
 
         doc = doc + "\n\n" + f["doc"]
-        plugins = [
+        plugins: List["PluginName"] = [
             "strikethrough",
             "footnotes",
             "table",
             "url",
-        ]  # type: List[PluginName]  # fix error Generic str != explicit Literals
+        ]
         # if escape active, wraps literal HTML into '<p> {HTML} </p>'
         # we must pass it to both since 'MyRenderer' is predefined
         escape = False
-        markdown2html = create_markdown(
+        markdown2html: "Markdown[str, Any]" = create_markdown(
             renderer=MyRenderer(escape=escape),
             plugins=plugins,
             escape=escape,
-        )  # type: Markdown[str, Any]
+        )
         markdown2html.before_parse_hooks.append(markdown_list_hook)
         doc = markdown2html(doc)
 

schema_salad/metaschema.py

@@ -429,8 +429,7 @@ def __repr__(self):  # type: () -> str
 
 
 class _EnumLoader(_Loader):
-    def __init__(self, symbols, name):
-        # type: (Sequence[str], str) -> None
+    def __init__(self, symbols: Sequence[str], name: str) -> None:
         self.symbols = symbols
         self.name = name
 
@@ -547,8 +546,7 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None):
 
 
 class _UnionLoader(_Loader):
-    def __init__(self, alternates):
-        # type: (Sequence[_Loader]) -> None
+    def __init__(self, alternates: Sequence[_Loader]) -> None:
         self.alternates = alternates
 
     def load(self, doc, baseuri, loadingOptions, docRoot=None):

schema_salad/python_codegen.py

@@ -377,7 +377,8 @@ def end_class(self, classname: str, field_names: List[str]) -> None:
         )
         if self.idfield:
             self.out.write(
-                f"        loadingOptions.idx[{self.safe_name(self.idfield)}] = (_constructed, loadingOptions)\n"
+                f"        loadingOptions.idx[{self.safe_name(self.idfield)}] "
+                "= (_constructed, loadingOptions)\n"
             )
 
         self.out.write("        return _constructed\n")

schema_salad/python_codegen_support.py

@@ -426,8 +426,7 @@ def __repr__(self):  # type: () -> str
 
 
 class _EnumLoader(_Loader):
-    def __init__(self, symbols, name):
-        # type: (Sequence[str], str) -> None
+    def __init__(self, symbols: Sequence[str], name: str) -> None:
         self.symbols = symbols
         self.name = name
 
@@ -544,8 +543,7 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None):
 
 
 class _UnionLoader(_Loader):
-    def __init__(self, alternates):
-        # type: (Sequence[_Loader]) -> None
+    def __init__(self, alternates: Sequence[_Loader]) -> None:
         self.alternates = alternates
 
     def load(self, doc, baseuri, loadingOptions, docRoot=None):

schema_salad/ref_resolver.py

@@ -310,7 +310,6 @@ def add_schemas(self, ns: Union[List[str], str], base_url: str) -> None:
                         break
                     except (xml.sax.SAXParseException, TypeError, BadSyntax) as e:
                         err_msg = str(e)
-                        pass
                 else:
                     _logger.warning(
                         "Could not load extension schema %s: %s", fetchurl, err_msg

schema_salad/schema.py

@@ -369,7 +369,7 @@ def validate_doc(
                 break
 
         if not success:
-            errors = []  # type: List[SchemaSaladException]
+            errors: List[SchemaSaladException] = []
             for root in roots:
                 if hasattr(root, "get_prop"):
                     name = root.get_prop("name")

schema_salad/tests/test_cli_args.py

@@ -20,7 +20,7 @@ def captured_output() -> Iterator[Tuple[StringIO, StringIO]]:
 
 
 def test_version() -> None:
-    args = [["--version"], ["-v"]]  # type: List[List[str]]
+    args: List[List[str]] = [["--version"], ["-v"]]
     for arg in args:
         with captured_output() as (out, err):
             cli_parser.main(arg)
@@ -31,7 +31,7 @@ def test_version() -> None:
 
 def test_empty_input() -> None:
     # running schema_salad tool without any args
-    args = []  # type: List[str]
+    args: List[str] = []
     with captured_output() as (out, err):
         cli_parser.main(args)
 

schema_salad/tests/test_cpp_codegen.py

@@ -1,23 +1,19 @@
 """Test C++ code generation."""
 
 import os
-import subprocess
 from pathlib import Path
 from typing import Any, Dict, List, cast
 
 from schema_salad import codegen
 from schema_salad.avro.schema import Names
 from schema_salad.schema import load_schema
-from schema_salad.sourceline import cmap
-from schema_salad.utils import yaml_no_ts
 
 from .util import cwl_file_uri, get_data
 
 
 def test_cwl_cpp_gen(tmp_path: Path) -> None:
     """End to end test of C++ generator using the CWL v1.0 schema."""
     src_target = tmp_path / "cwl_v1_0.h"
-    exe_target = tmp_path / "cwl_v1_0_test"
     cpp_codegen(cwl_file_uri, src_target)
     source = get_data("tests/codegen/cwl.cpp")
     assert source