👌 fix quadratic complexity in reference parser (#367)

ports: markdown-it/markdown-it@de814ca

Author: chrisjsewell

docs/conf.py

@@ -52,6 +52,7 @@
         ".*_NodeType",
         ".*Literal.*",
         ".*_Result",
+        ".*_State",
         "EnvType",
         "Path",
         "Ellipsis",

markdown_it/helpers/parse_link_destination.py

@@ -6,17 +6,15 @@
 
 
 class _Result:
-    __slots__ = ("lines", "ok", "pos", "str")
+    __slots__ = ("ok", "pos", "str")
 
     def __init__(self) -> None:
         self.ok = False
         self.pos = 0
-        self.lines = 0
         self.str = ""
 
 
 def parseLinkDestination(string: str, pos: int, maximum: int) -> _Result:
-    lines = 0
     start = pos
     result = _Result()
 
@@ -80,7 +78,6 @@ def parseLinkDestination(string: str, pos: int, maximum: int) -> _Result:
         return result
 
     result.str = unescapeAll(string[start:pos])
-    result.lines = lines
     result.pos = pos
     result.ok = True
     return result

markdown_it/helpers/parse_link_title.py

@@ -3,58 +3,73 @@
 from ..common.utils import charCodeAt, unescapeAll
 
 
-class _Result:
-    __slots__ = ("lines", "ok", "pos", "str")
+class _State:
+    __slots__ = ("can_continue", "marker", "ok", "pos", "str")
 
     def __init__(self) -> None:
         self.ok = False
+        """if `true`, this is a valid link title"""
+        self.can_continue = False
+        """if `true`, this link can be continued on the next line"""
         self.pos = 0
-        self.lines = 0
+        """if `ok`, it's the position of the first character after the closing marker"""
         self.str = ""
+        """if `ok`, it's the unescaped title"""
+        self.marker = 0
+        """expected closing marker character code"""
 
     def __str__(self) -> str:
         return self.str
 
 
-def parseLinkTitle(string: str, pos: int, maximum: int) -> _Result:
-    lines = 0
-    start = pos
-    result = _Result()
+def parseLinkTitle(
+    string: str, start: int, maximum: int, prev_state: _State | None = None
+) -> _State:
+    """Parse link title within `str` in [start, max] range,
+    or continue previous parsing if `prev_state` is defined (equal to result of last execution).
+    """
+    pos = start
+    state = _State()
 
-    if pos >= maximum:
-        return result
+    if prev_state is not None:
+        # this is a continuation of a previous parseLinkTitle call on the next line,
+        # used in reference links only
+        state.str = prev_state.str
+        state.marker = prev_state.marker
+    else:
+        if pos >= maximum:
+            return state
 
-    marker = charCodeAt(string, pos)
+        marker = charCodeAt(string, pos)
 
-    # /* " */  /* ' */  /* ( */
-    if marker != 0x22 and marker != 0x27 and marker != 0x28:
-        return result
+        # /* " */  /* ' */  /* ( */
+        if marker != 0x22 and marker != 0x27 and marker != 0x28:
+            return state
 
-    pos += 1
+        start += 1
+        pos += 1
+
+        # if opening marker is "(", switch it to closing marker ")"
+        if marker == 0x28:
+            marker = 0x29
 
-    # if opening marker is "(", switch it to closing marker ")"
-    if marker == 0x28:
-        marker = 0x29
+        state.marker = marker
 
     while pos < maximum:
         code = charCodeAt(string, pos)
-        if code == marker:
-            title = string[start + 1 : pos]
-            title = unescapeAll(title)
-            result.pos = pos + 1
-            result.lines = lines
-            result.str = title
-            result.ok = True
-            return result
-        elif code == 0x28 and marker == 0x29:  # /* ( */  /* ) */
-            return result
-        elif code == 0x0A:
-            lines += 1
+        if code == state.marker:
+            state.pos = pos + 1
+            state.str += unescapeAll(string[start:pos])
+            state.ok = True
+            return state
+        elif code == 0x28 and state.marker == 0x29:  # /* ( */  /* ) */
+            return state
         elif code == 0x5C and pos + 1 < maximum:  # /* \ */
             pos += 1
-            if charCodeAt(string, pos) == 0x0A:
-                lines += 1
 
         pos += 1
 
-    return result
+    # no closing marker found, but this link title may continue on the next line (for references)
+    state.can_continue = True
+    state.str += unescapeAll(string[start:pos])
+    return state

markdown_it/rules_block/reference.py

@@ -11,7 +11,6 @@ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) ->
         "entering reference: %s, %s, %s, %s", state, startLine, _endLine, silent
     )
 
-    lines = 0
     pos = state.bMarks[startLine] + state.tShift[startLine]
     maximum = state.eMarks[startLine]
     nextLine = startLine + 1
@@ -22,51 +21,9 @@ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) ->
     if state.src[pos] != "[":
         return False
 
-    # Simple check to quickly interrupt scan on [link](url) at the start of line.
-    # Can be useful on practice: https:#github.com/markdown-it/markdown-it/issues/54
-    while pos < maximum:
-        # /* ] */  /* \ */  /* : */
-        if state.src[pos] == "]" and state.src[pos - 1] != "\\":
-            if pos + 1 == maximum:
-                return False
-            if state.src[pos + 1] != ":":
-                return False
-            break
-        pos += 1
-
-    endLine = state.lineMax
-
-    # jump line-by-line until empty one or EOF
-    terminatorRules = state.md.block.ruler.getRules("reference")
+    string = state.src[pos : maximum + 1]
 
-    oldParentType = state.parentType
-    state.parentType = "reference"
-
-    while nextLine < endLine and not state.isEmpty(nextLine):
-        # this would be a code block normally, but after paragraph
-        # it's considered a lazy continuation regardless of what's there
-        if state.sCount[nextLine] - state.blkIndent > 3:
-            nextLine += 1
-            continue
-
-        # quirk for blockquotes, this line should already be checked by that rule
-        if state.sCount[nextLine] < 0:
-            nextLine += 1
-            continue
-
-        # Some tags can terminate paragraph without empty line.
-        terminate = False
-        for terminatorRule in terminatorRules:
-            if terminatorRule(state, nextLine, endLine, True):
-                terminate = True
-                break
-
-        if terminate:
-            break
-
-        nextLine += 1
-
-    string = state.getLines(startLine, nextLine, state.blkIndent, False).strip()
+    # string = state.getLines(startLine, nextLine, state.blkIndent, False).strip()
     maximum = len(string)
 
     labelEnd = None
@@ -79,11 +36,20 @@ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) ->
             labelEnd = pos
             break
         elif ch == 0x0A:  # /* \n */
-            lines += 1
+            if (lineContent := getNextLine(state, nextLine)) is not None:
+                string += lineContent
+                maximum = len(string)
+                nextLine += 1
         elif ch == 0x5C:  # /* \ */
             pos += 1
-            if pos < maximum and charCodeAt(string, pos) == 0x0A:
-                lines += 1
+            if (
+                pos < maximum
+                and charCodeAt(string, pos) == 0x0A
+                and (lineContent := getNextLine(state, nextLine)) is not None
+            ):
+                string += lineContent
+                maximum = len(string)
+                nextLine += 1
         pos += 1
 
     if (
@@ -97,7 +63,10 @@ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) ->
     while pos < maximum:
         ch = charCodeAt(string, pos)
         if ch == 0x0A:
-            lines += 1
+            if (lineContent := getNextLine(state, nextLine)) is not None:
+                string += lineContent
+                maximum = len(string)
+                nextLine += 1
         elif isSpace(ch):
             pass
         else:
@@ -106,28 +75,30 @@ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) ->
 
     # [label]:   destination   'title'
     #            ^^^^^^^^^^^ parse this
-    res = state.md.helpers.parseLinkDestination(string, pos, maximum)
-    if not res.ok:
+    destRes = state.md.helpers.parseLinkDestination(string, pos, maximum)
+    if not destRes.ok:
         return False
 
-    href = state.md.normalizeLink(res.str)
+    href = state.md.normalizeLink(destRes.str)
     if not state.md.validateLink(href):
         return False
 
-    pos = res.pos
-    lines += res.lines
+    pos = destRes.pos
 
     # save cursor state, we could require to rollback later
     destEndPos = pos
-    destEndLineNo = lines
+    destEndLineNo = nextLine
 
     # [label]:   destination   'title'
     #                       ^^^ skipping those spaces
     start = pos
     while pos < maximum:
         ch = charCodeAt(string, pos)
         if ch == 0x0A:
-            lines += 1
+            if (lineContent := getNextLine(state, nextLine)) is not None:
+                string += lineContent
+                maximum = len(string)
+                nextLine += 1
         elif isSpace(ch):
             pass
         else:
@@ -136,15 +107,23 @@ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) ->
 
     # [label]:   destination   'title'
     #                          ^^^^^^^ parse this
-    res = state.md.helpers.parseLinkTitle(string, pos, maximum)
-    if pos < maximum and start != pos and res.ok:
-        title = res.str
-        pos = res.pos
-        lines += res.lines
+    titleRes = state.md.helpers.parseLinkTitle(string, pos, maximum, None)
+    while titleRes.can_continue:
+        if (lineContent := getNextLine(state, nextLine)) is None:
+            break
+        string += lineContent
+        pos = maximum
+        maximum = len(string)
+        nextLine += 1
+        titleRes = state.md.helpers.parseLinkTitle(string, pos, maximum, titleRes)
+
+    if pos < maximum and start != pos and titleRes.ok:
+        title = titleRes.str
+        pos = titleRes.pos
     else:
         title = ""
         pos = destEndPos
-        lines = destEndLineNo
+        nextLine = destEndLineNo
 
     # skip trailing spaces until the rest of the line
     while pos < maximum:
@@ -158,7 +137,7 @@ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) ->
         # but it could still be a valid reference if we roll back
         title = ""
         pos = destEndPos
-        lines = destEndLineNo
+        nextLine = destEndLineNo
         while pos < maximum:
             ch = charCodeAt(string, pos)
             if not isSpace(ch):
@@ -181,7 +160,7 @@ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) ->
     if "references" not in state.env:
         state.env["references"] = {}
 
-    state.line = startLine + lines + 1
+    state.line = nextLine
 
     # note, this is not part of markdown-it JS, but is useful for renderers
     if state.md.options.get("inline_definitions", False):
@@ -210,6 +189,47 @@ def reference(state: StateBlock, startLine: int, _endLine: int, silent: bool) ->
             }
         )
 
-    state.parentType = oldParentType
-
     return True
+
+
+def getNextLine(state: StateBlock, nextLine: int) -> None | str:
+    endLine = state.lineMax
+
+    if nextLine >= endLine or state.isEmpty(nextLine):
+        # empty line or end of input
+        return None
+
+    isContinuation = False
+
+    # this would be a code block normally, but after paragraph
+    # it's considered a lazy continuation regardless of what's there
+    if state.is_code_block(nextLine):
+        isContinuation = True
+
+    # quirk for blockquotes, this line should already be checked by that rule
+    if state.sCount[nextLine] < 0:
+        isContinuation = True
+
+    if not isContinuation:
+        terminatorRules = state.md.block.ruler.getRules("reference")
+        oldParentType = state.parentType
+        state.parentType = "reference"
+
+        # Some tags can terminate paragraph without empty line.
+        terminate = False
+        for terminatorRule in terminatorRules:
+            if terminatorRule(state, nextLine, endLine, True):
+                terminate = True
+                break
+
+        state.parentType = oldParentType
+
+        if terminate:
+            # terminated by another block
+            return None
+
+    pos = state.bMarks[nextLine] + state.tShift[nextLine]
+    maximum = state.eMarks[nextLine]
+
+    # max + 1 explicitly includes the newline
+    return state.src[pos : maximum + 1]

markdown_it/rules_inline/image.py

@@ -66,7 +66,7 @@ def image(state: StateInline, silent: bool) -> bool:
 
         # [link](  <href>  "title"  )
         #                  ^^^^^^^ parsing link title
-        res = state.md.helpers.parseLinkTitle(state.src, pos, state.posMax)
+        res = state.md.helpers.parseLinkTitle(state.src, pos, state.posMax, None)
         if pos < max and start != pos and res.ok:
             title = res.str
             pos = res.pos