WIP turning on some more mypy options and cleaning up the resultant mess

Author: tabatkins

bikeshed/caniuse/caniuse.py

@@ -60,7 +60,7 @@ def addCanIUsePanels(doc: t.SpecT) -> list[t.ElementT]:
     return panels
 
 
-def canIUsePanelFor(id: str, data: t.JSONT, update: str, classFromBrowser: dict[str, str]) -> t.ElementT:
+def canIUsePanelFor(id: str, data: t.JSONObject, update: str, classFromBrowser: dict[str, str]) -> t.ElementT:
     panel = h.E.details(
         {"class": "caniuse-status unpositioned", "data-deco": ""},
         h.E.summary({}, "CanIUse"),
@@ -157,19 +157,19 @@ def __init__(self, dataFile: t.DataFileRequester) -> None:
         self.updated = data["updated"]
         self.agents = data["agents"]
         self.urlFromFeature = data["features"]
-        self.features: t.JSONT = {}
+        self.features: t.JSONObject = {}
 
     def hasFeature(self, featureName: str) -> bool:
         return featureName in self.urlFromFeature
 
-    def getFeature(self, featureName: str) -> t.JSONT:
+    def getFeature(self, featureName: str) -> t.JSONObject:
         if featureName in self.features:
-            return t.cast("t.JSONT", self.features[featureName])
+            return t.cast("t.JSONObject", self.features[featureName])
         if not self.hasFeature(featureName):
             return {}
         data = json.loads(
             self.dataFile.fetch("caniuse", f"feature-{featureName}.json", str=True),
             object_pairs_hook=OrderedDict,
         )
         self.features[featureName] = data
-        return t.cast("t.JSONT", data)
+        return t.cast("t.JSONObject", data)

bikeshed/cddl.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import dataclasses
 from typing import get_args
 
 import cddlparser
@@ -8,6 +9,13 @@
 from . import messages as m
 
 
+@dataclasses.dataclass
+class CDDLTerm:
+    type: str
+    name: str
+    dfnFor: str | None
+
+
 class CDDLMarker(cddlparser.ast.Marker):
     """
     Marker that wraps CDDL definitions and references in <cddl> and <a> blocks
@@ -20,7 +28,7 @@ class CDDLMarker(cddlparser.ast.Marker):
     currentParameters: list[str]
 
     # List of all CDDL terms defined so far to track and report duplicates
-    defined: list
+    defined: list[CDDLTerm]
 
     def __init__(self) -> None:
         self.currentRule = None
@@ -29,21 +37,21 @@ def __init__(self) -> None:
 
     def _recordDefinition(self, type: str, name: str, dfnFor: str | None = None) -> bool:
         for term in self.defined:
-            if term["type"] == type and term["name"] == name and term["dfnFor"] == dfnFor:
+            if term.type == type and term.name == name and term.dfnFor == dfnFor:
                 forText = "" if dfnFor is None else f' defined in type "{dfnFor}"'
                 m.die(
                     f"CDDL {type} {name}{forText} creates a duplicate and cannot be referenced.\nPlease create additional CDDL types to disambiguate.",
                 )
                 return False
         if type != "parameter":
             for term in self.defined:
-                if term["type"] != "parameter" and term["name"] == name and term["dfnFor"] == dfnFor:
+                if term.type != "parameter" and term.name == name and term.dfnFor == dfnFor:
                     forText = "" if dfnFor is None else f' defined in type "{dfnFor}"'
                     m.warn(
-                        f"CDDL {type} {name}{forText} creates a duplicate with a CDDL {term['type']}.\nLink type needs to be specified to reference the term.\nConsider creating additional CDDL types to disambiguate.",
+                        f"CDDL {type} {name}{forText} creates a duplicate with a CDDL {term.type}.\nLink type needs to be specified to reference the term.\nConsider creating additional CDDL types to disambiguate.",
                     )
                     break
-        term = {"type": type, "name": name, "dfnFor": dfnFor}
+        term = CDDLTerm(type=type, name=name, dfnFor=dfnFor)
         self.defined.append(term)
         return True
 

bikeshed/config/BoolSet.py

@@ -5,7 +5,7 @@
 from .. import t
 
 
-class BoolSet(collections.abc.MutableMapping):
+class BoolSet(collections.abc.MutableMapping[str, bool]):
     """
     Implements a "boolean set",
     where keys can be explicitly set to True or False,

bikeshed/config/__init__.py

@@ -29,7 +29,6 @@
     docPath,
     doEvery,
     englishFromList,
-    flatten,
     groupFromKey,
     intersperse,
     processTextNodes,

bikeshed/config/dfnTypes.py

@@ -221,7 +221,7 @@ def linkTypeIn(startTypes: str | t.AbstractSet[str], targetTypes: str | t.Abstra
 
 # Some of the more significant types and their patterns
 trivialPattern = re.compile(r".+")
-typeRe: dict[str, re.Pattern]
+typeRe: dict[str, re.Pattern[str]]
 typeRe = defaultdict(lambda: trivialPattern)
 typeRe["property"] = re.compile(r"^[\w-]+$")
 typeRe["at-rule"] = re.compile(r"^@[\w-]+$")

bikeshed/config/main.py

@@ -21,15 +21,10 @@ def englishFromList(items: t.Iterable[str], conjunction: str = "or") -> str:
     return "{0}, {2} {1}".format(", ".join(items[:-1]), items[-1], conjunction)
 
 
-if t.TYPE_CHECKING:
-    IntersperseU = t.TypeVar("IntersperseU")
-    IntersperseV = t.TypeVar("IntersperseV")
-
-
-def intersperse(
-    iterable: t.Iterable[IntersperseU],
-    delimiter: IntersperseV,
-) -> t.Generator[IntersperseU | IntersperseV, None, None]:
+def intersperse[IterT, DelimT](
+    iterable: t.Iterable[IterT],
+    delimiter: DelimT,
+) -> t.Iterator[IterT | DelimT]:
     first = True
     for x in iterable:
         if not first:
@@ -38,13 +33,13 @@ def intersperse(
         yield x
 
 
-if t.TYPE_CHECKING:
-    ProcessTextNodesU = t.TypeVar("ProcessTextNodesU")
-
-
-def processTextNodes(nodes: list[t.NodeT], regex: re.Pattern, replacer: t.Callable) -> list[t.NodeT]:
+def processTextNodes(
+    nodes: t.Iterable[t.NodeT],
+    regex: t.Pattern,
+    replacer: t.Callable[[t.Match], t.NodeT],
+) -> list[t.NodeT]:
     """
-    Takes an array of text/objects,
+    Takes an array of nodes,
     and flatmaps reSubObject over the text parts.
     """
     ret: list[t.NodeT] = []
@@ -56,35 +51,35 @@ def processTextNodes(nodes: list[t.NodeT], regex: re.Pattern, replacer: t.Callab
     return ret
 
 
-if t.TYPE_CHECKING:
-    ReSubObjectU = t.TypeVar("ReSubObjectU")
-
-
 @t.overload
-def reSubObject(pattern: re.Pattern, string: str, repl: None) -> list[str | re.Match]: ...
+def reSubObject(
+    pattern: t.Pattern,
+    string: str,
+    repl: None = None,
+) -> list[str | t.Match]: ...
 
 
 @t.overload
-def reSubObject(
-    pattern: re.Pattern,
+def reSubObject[SubT](
+    pattern: t.Pattern,
     string: str,
-    repl: t.Callable[[re.Match], ReSubObjectU],
-) -> list[str | ReSubObjectU]: ...
+    repl: t.Callable[[t.Match], SubT],
+) -> list[str | SubT]: ...
 
 
-def reSubObject(
-    pattern: re.Pattern,
+def reSubObject[SubT](
+    pattern: t.Pattern,
     string: str,
-    repl: t.Callable[[re.Match], ReSubObjectU] | None = None,
-) -> list[str | re.Match] | list[str | ReSubObjectU] | list[str | re.Match | ReSubObjectU]:
+    repl: t.Callable[[t.Match], SubT] | None = None,
+) -> list[str | t.Match] | list[str | SubT] | list[str | t.Match | SubT]:
     """
     like re.sub, but replacements don't have to be text;
     returns an array of alternating unmatched text and match objects instead.
     If repl is specified, it's called with each match object,
     and the result then shows up in the array instead.
     """
     lastEnd = 0
-    pieces: list[str | re.Match | ReSubObjectU] = []
+    pieces: list[str | t.Match | SubT] = []
     for match in pattern.finditer(string):
         pieces.append(string[lastEnd : match.start()])
         if repl:
@@ -150,14 +145,6 @@ def groupFromKey(key: str, length: int = 2) -> str:
 _groupFromKeyCache = {}
 
 
-def flatten(arr: t.Iterable) -> t.Generator:
-    for el in arr:
-        if isinstance(el, collections.abc.Iterable) and not isinstance(el, str) and not lxml.etree.iselement(el):
-            yield from flatten(el)
-        else:
-            yield el
-
-
 def scriptPath(*pathSegs: str) -> str:
     startPath = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
     path = os.path.join(startPath, *pathSegs)
@@ -183,7 +170,7 @@ def chrootPath(rootPath: str, path: str) -> str:
         return path
 
 
-def doEvery(s: float, action: t.Callable, lastTime: float | None = None) -> float:
+def doEvery(s: float, action: t.Callable[[], t.Any], lastTime: float | None = None) -> float:
     # Takes an action every N seconds.
     # Pass it the duration and the last time it took the action;
     # it returns the time it last took the action
@@ -204,7 +191,19 @@ def doEvery(s: float, action: t.Callable, lastTime: float | None = None) -> floa
     SafeIndexDefaultT = t.TypeVar("SafeIndexDefaultT")
 
 
-def safeIndex(coll: t.Sequence, needle: t.Any, default: SafeIndexDefaultT = None) -> int | SafeIndexDefaultT:  # type: ignore[assignment]
+@t.overload
+def safeIndex[ValT](coll: t.Sequence[ValT], needle: ValT) -> int | None: ...
+
+
+@t.overload
+def safeIndex[ValT, DefaultT](coll: t.Sequence[ValT], needle: ValT, default: DefaultT) -> int | DefaultT: ...
+
+
+def safeIndex[ValT, DefaultT](
+    coll: t.Sequence[ValT],
+    needle: ValT,
+    default: DefaultT | None = None,
+) -> int | DefaultT | None:
     try:
         return coll.index(needle)
     except ValueError:

bikeshed/datablocks.py

@@ -893,7 +893,7 @@ def extendData(datas: InfoTreeT, infoLevels: InfoTreeT) -> None:
             # HTML comment filling the whole line,
             # go ahead and strip it
             continue
-        ws, text = t.cast("re.Match", re.match(r"(\s*)(.*)", line)).groups()
+        ws, text = t.cast("t.Match", re.match(r"(\s*)(.*)", line)).groups()
         if text.startswith("#"):  # comment
             continue
         wsLen = len(ws.replace("\t", indentSpace))

bikeshed/h/dom.py

@@ -19,9 +19,9 @@
     from .parser import StartTag
 
 
-def flatten(arr: t.Iterable) -> t.Iterator:
+def flatten(arr: t.NodesT | tuple[t.NodesT, ...]) -> t.Iterator[t.NodeT]:
     for el in arr:
-        if isinstance(el, collections.abc.Iterable) and not isinstance(el, str) and not etree.iselement(el):
+        if isinstance(el, collections.abc.Iterable) and not isinstance(el, str) and not lxml.etree.iselement(el):
             yield from flatten(el)
         else:
             yield el
@@ -535,7 +535,7 @@ def replaceContents(el: t.ElementT, newElements: t.NodesT | t.Iterable[t.NodesT]
 
 
 def replaceWithContents(el: t.ElementT) -> t.NodesT | None:
-    return replaceNode(el, childNodes(el, clear=True))
+    return replaceNode(el, t.cast("t.NodesT", childNodes(el, clear=True)))
 
 
 def moveContents(toEl: t.ElementT, fromEl: t.ElementT) -> None:
@@ -1006,8 +1006,8 @@ def replaceMacrosTextly(text: str, macros: t.Mapping[str, str], context: str) ->
     # Same as replaceMacros(), but does the substitution
     # directly on the text, rather than relying on the
     # html parser to have preparsed the macro syntax
-    def macroReplacer(match: re.Match) -> str:
-        fullText = t.cast(str, match.group(0))
+    def macroReplacer(match: t.Match) -> str:
+        fullText = match.group(0)
         innerText = match.group(2).lower() or ""
         optional = match.group(3) == "?"
         if fullText.startswith("\\"):

bikeshed/h/merge.py

@@ -38,7 +38,7 @@ class TagStr(Tag):
     TagStream: t.TypeAlias = t.Generator[Tag, None, None]
 
 
-def mergeTrees(tree1: t.ElementT, tree2: t.ElementT) -> list:
+def mergeTrees(tree1: t.ElementT, tree2: t.ElementT) -> list[t.NodeT]:
     """
     Attempts to merge two HTML trees
     of the same text content.

bikeshed/h/parser/parser.py

@@ -2410,7 +2410,7 @@ def parseLinkInfo(
     innerStart: int,
     startSigil: str,
     endSigil: str,
-    infoRe: re.Pattern,
+    infoRe: t.Pattern,
 ) -> ResultT[LinkData]:
     start = innerStart - len(startSigil)
     match, innerEnd, _ = s.searchRe(innerStart, infoRe)
@@ -2518,11 +2518,11 @@ def parseBiblioInner(s: Stream, innerStart: int) -> ResultT[tuple[StartTag, str]
     normative = match[1] == "!"
     lt = match[2]
     modifierSequence = match[3].strip()
-    attrs = {
-        "data-lt": lt,
-        "data-link-type": "biblio",
-        "data-biblio-type": "normative" if normative else "informative",
-        "bs-autolink-syntax": s.slice(nodeStart, innerEnd) + "]]",
+    attrs: t.SafeAttrDict = {
+        "data-lt": escapeAttr(lt),
+        "data-link-type": t.SafeAttrStr("biblio"),
+        "data-biblio-type": t.SafeAttrStr("normative" if normative else "informative"),
+        "bs-autolink-syntax": escapeAttr(s.slice(nodeStart, innerEnd) + "]]"),
     }
 
     failureStart = StartTag.fromStream(s, nodeStart, innerStart, "span")
@@ -2535,7 +2535,7 @@ def parseBiblioInner(s: Stream, innerStart: int) -> ResultT[tuple[StartTag, str]
         for modifier in re.split(r"\s+", modifierSequence):
             if modifier in ("current", "snapshot"):
                 if "data-biblio-status" not in attrs:
-                    attrs["data-biblio-status"] = modifier
+                    attrs["data-biblio-status"] = t.SafeAttrStr(modifier)
                 else:
                     m.die(
                         f"Biblio shorthand [{lt} ...] contains multiple current/snapshot keywords. Please use only one.",
@@ -2544,7 +2544,7 @@ def parseBiblioInner(s: Stream, innerStart: int) -> ResultT[tuple[StartTag, str]
                     return failureResult
             elif modifier in ("inline", "index", "direct"):
                 if "data-biblio-display" not in attrs:
-                    attrs["data-biblio-display"] = modifier
+                    attrs["data-biblio-display"] = t.SafeAttrStr(modifier)
                 else:
                     m.die(
                         f"Biblio shorthand [{lt} ...] contains multiple inline/index/direct keywords. Please use only one.",
@@ -2876,15 +2876,15 @@ def replaceMacrosInText(text: str, macros: dict[str, str], s: Stream, start: int
     msc = constants.macroStartChar
     mec = constants.macroEndChar
 
-    def doRep(match: re.Match) -> str:
+    def doRep(match: t.Match) -> str:
         escaped = match[1] == "\\"
         biblio = match[1] == "["
         macroName = match[2].lower()
         optional = match[3] == "?"
         if escaped:
-            return msc + "[" + t.cast("str", match[0][2:-1]) + "]" + mec
+            return msc + "[" + match[0][2:-1] + "]" + mec
         if biblio:
-            return msc + "[[" + t.cast("str", match[0][2:-1]) + "]" + mec
+            return msc + "[[" + match[0][2:-1] + "]" + mec
         if macroName in macros:
             return msc + macros[macroName] + mec
         if optional:
@@ -2893,7 +2893,7 @@ def doRep(match: re.Match) -> str:
             f"Found unmatched text macro {match[0]} in {context}. Correct the macro, or escape it by replacing the opening [ with &bs[;.",
             lineNum=s.loc(start),
         )
-        return msc + "[" + t.cast("str", match[0][1:-1]) + "]" + mec
+        return msc + "[" + match[0][1:-1] + "]" + mec
 
     text, count = MACRO_ATTRIBUTE_RE.subn(doRep, text)
     if count > 0:
@@ -3305,5 +3305,5 @@ def htmlifyMarkdownEscapes(val: str) -> str:
     return re.sub(MARKDOWN_ESCAPE_RE, markdownEscapeReplacer, val)
 
 
-def markdownEscapeReplacer(match: re.Match) -> str:
+def markdownEscapeReplacer(match: t.Match) -> str:
     return f"&#{ord(match[1])};"