feat: markdown dialect option, faster search (0.1.1)

- split_markdown/load_markdown: dialect="commonmark"|"python-markdown" so list
  blocks match the HTML structure of the renderer (list after paragraph needs a
  blank line; nesting needs 4 spaces in Python-Markdown)
- SearchEngine caches the searched text per block on load
- regex timeout only for /regex/ terms (words and phrases are linear);
  typical queries over 26k blocks: ~30-140 ms instead of 250-700 ms

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -12,8 +12,8 @@ Boolean full-text search over Markdown (and any other text) – as a **Python li
 ## Installation
 
 ```bash
-pip install "mdgrep @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.0"        # library + CLI
+pip install "mdgrep @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.1"        # library + CLI
-pip install "mdgrep[web] @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.0"   # + Flask API
+pip install "mdgrep[web] @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.1"   # + Flask API
 ```
 
 Requires Python ≥ 3.10. Dependencies: [`regex`](https://pypi.org/project/regex/) (and `flask` for `[web]`).
@@ -97,6 +97,10 @@ engine.search("login OR sso", filters={"status": "open"})
 `load_markdown()` adds `meta["section"]` (nearest heading), `meta["kind"]`
 (`file`/`section`/`item`/`paragraph`) and `meta["markdown"]` (raw source).
 
+`dialect` (for `split="list"`) makes blocks line up with the HTML of your Markdown renderer:
+`"commonmark"` (default) or `"python-markdown"` (a list needs a blank line before it; nesting
+needs 4 spaces of indentation). Important if you link results to element ids in rendered HTML.
+
 ### `SearchEngine(source, *, reload_check=None, reload_interval=60)`
 
 `source` is a list of blocks or a callable returning them. With a callable plus `reload_check`

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mdgrep"
-version = "0.1.0"
+version = "0.1.1"
 description = "Boolean full-text search (substrings, phrases, regex, AND/OR/NOT, parentheses) over Markdown and other text blocks, with CLI, Flask API and a JavaScript search widget"
 readme = "README.md"
 license = "MIT"

src/mdgrep/__init__.py

@@ -5,7 +5,7 @@ from .markdown import load_markdown, markdown_to_text, split_markdown
 from .query import Query, QuerySyntaxError, QueryTimeout, compile_query
 from .snippets import segments
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"
 
 STATIC_FILES = ("mdgrep-widget.js", "mdgrep-widget.css")
 

src/mdgrep/engine.py

@@ -103,12 +103,17 @@ class SearchEngine:
         self._lock = threading.Lock()
         self._token = reload_check() if reload_check else None
         self._checked_at = time.monotonic()
-        self._blocks: list[Block] = list(source() if callable(source) else source)
+        self._set_blocks(list(source() if callable(source) else source))
+
+    def _set_blocks(self, blocks: list[Block]) -> None:
+        # Text that is searched (title + text), precomputed once per load.
+        haystacks = [f"{b.title}\n{b.text}" if b.title else b.text for b in blocks]
+        self._data = (blocks, haystacks)
 
     @property
     def blocks(self) -> list[Block]:
         self._maybe_reload()
-        return self._blocks
+        return self._data[0]
 
     def reload(self) -> None:
         """Reload blocks from the loader (no-op for a static block list)."""
@@ -116,7 +121,7 @@ class SearchEngine:
             return
         blocks = list(self._loader())
         with self._lock:
-            self._blocks = blocks
+            self._set_blocks(blocks)
             self._token = self._reload_check() if self._reload_check else None
             self._checked_at = time.monotonic()
 
@@ -160,14 +165,19 @@ class SearchEngine:
         started = time.monotonic()
         q = query if isinstance(query, Query) else compile_query(query)
         deadline = started + time_budget if time_budget else None
-        candidates = self._filter(self.blocks, filters, predicate)
+        self._maybe_reload()
+        blocks, haystacks = self._data
+        conditions = self._conditions(filters)
 
         matched: list[Block] = []
         truncated = False
-        for block in candidates:
+        for i, block in enumerate(blocks):
+            if conditions and any(block.meta.get(k) not in allowed for k, allowed in conditions):
+                continue
+            if predicate is not None and not predicate(block):
+                continue
             try:
-                if q.matches(f"{block.title}\n{block.text}" if block.title else block.text,
+                if q.matches(haystacks[i], deadline=deadline):
-                             deadline=deadline):
                     matched.append(block)
             except QueryTimeout:
                 truncated = True
@@ -214,16 +224,11 @@ class SearchEngine:
         )
 
     @staticmethod
-    def _filter(blocks, filters, predicate):
+    def _conditions(filters):
         conditions = []
         for key, value in (filters or {}).items():
             if value is None or value == "" or value == []:
                 continue
             allowed = set(value) if isinstance(value, (list, tuple, set, frozenset)) else {value}
             conditions.append((key, allowed))
-        for block in blocks:
+        return conditions
-            if any(block.meta.get(k) not in allowed for k, allowed in conditions):
-                continue
-            if predicate is not None and not predicate(block):
-                continue
-            yield block

src/mdgrep/markdown.py

@@ -9,6 +9,13 @@ Split modes:
 
 Every block gets `meta["section"]` (nearest preceding heading text), `meta["kind"]`
 ("file", "section", "item" or "paragraph") and `meta["markdown"]` (the raw source).
+`dialect` (list mode) decides how list items are recognised, so blocks line up with the HTML
+your Markdown renderer produces:
+    commonmark       (default) a list may interrupt a paragraph; only unindented markers start
+                     top-level items
+    python-markdown  a list needs a blank line before it; markers indented by up to 3 spaces
+                     still start top-level items (nesting needs 4 spaces)
+
 The searchable `text` is plain text with Markdown syntax removed. In `file` and `heading`
 mode the heading becomes the block title (and is removed from the text); list items and
 paragraphs have no title.
@@ -22,19 +29,22 @@ from typing import Callable, Iterable
 
 from .engine import Block
 
-__all__ = ["load_markdown", "split_markdown", "markdown_to_text", "slugify", "SPLIT_MODES"]
+__all__ = ["load_markdown", "split_markdown", "markdown_to_text", "slugify", "SPLIT_MODES", "DIALECTS"]
 
 SPLIT_MODES = ("file", "heading", "list", "paragraph")
+DIALECTS = ("commonmark", "python-markdown")
 
 _HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
 _FENCE = re.compile(r"^\s*(```|~~~)")
 _LIST_ITEM = re.compile(r"^([-*+]|\d{1,9}[.)])\s+")
+_LIST_ITEM_PYMD = re.compile(r"^ {0,3}([-*+]|\d{1,9}\.)\s+")
 _HR = re.compile(r"^\s*([-*_])(\s*\1){2,}\s*$")
 
 
 def load_markdown(paths: str | Path | Iterable[str | Path], *, split: str = "paragraph",
                   meta: Callable[[Path], dict] | None = None, encoding: str = "utf-8",
-                  pattern: str = "*.md", relative_to: str | Path | None = None) -> list[Block]:
+                  pattern: str = "*.md", relative_to: str | Path | None = None,
+                  dialect: str = "commonmark") -> list[Block]:
     """Load Markdown files (directories are searched recursively for `pattern`).
 
     `meta(path)` may return a dict merged into every block's meta of that file.
@@ -47,7 +57,7 @@ def load_markdown(paths: str | Path | Iterable[str | Path], *, split: str = "par
         text = path.read_text(encoding=encoding)
         extra = meta(path) if meta else {}
         source = path.resolve().relative_to(base).as_posix() if base else str(path)
-        blocks.extend(split_markdown(text, source=source, split=split, meta=extra))
+        blocks.extend(split_markdown(text, source=source, split=split, meta=extra, dialect=dialect))
     return blocks
 
 
@@ -64,18 +74,19 @@ def _expand(paths, pattern):
 
 
 def split_markdown(text: str, *, source: str = "", split: str = "paragraph",
-                   meta: dict | None = None) -> list[Block]:
+                   meta: dict | None = None, dialect: str = "commonmark") -> list[Block]:
-    """Split one Markdown document into blocks (see module docstring for modes)."""
+    """Split one Markdown document into blocks (see module docstring for modes and dialects)."""
     if split not in SPLIT_MODES:
         raise ValueError(f"split must be one of {SPLIT_MODES}, got {split!r}")
+    if dialect not in DIALECTS:
+        raise ValueError(f"dialect must be one of {DIALECTS}, got {dialect!r}")
     meta = meta or {}
     lines = text.splitlines()
-    raw_blocks = {
+    if split == "list":
-        "file": _split_file,
+        raw_blocks = _split_list(lines, dialect == "python-markdown")
-        "heading": _split_heading,
+    else:
-        "list": _split_list,
+        raw_blocks = {"file": _split_file, "heading": _split_heading,
-        "paragraph": _split_paragraph,
+                      "paragraph": _split_paragraph}[split](lines)
-    }[split](lines)
 
     blocks = []
     for start, end, kind, section, title, anchor in raw_blocks:
@@ -127,7 +138,7 @@ def _split_heading(lines):
     return out
 
 
-def _split_list(lines):
+def _split_list(lines, pymd=False):
     out = []
     section = ""
     cur = None           # [start, kind]
@@ -139,6 +150,7 @@ def _split_list(lines):
             out.append((cur[0], end, cur[1], section, "", ""))
             cur = None
 
+    item_re = _LIST_ITEM_PYMD if pymd else _LIST_ITEM
     prev_blank = True
     for i, line, in_fence, heading in _scan(lines):
         if heading:
@@ -154,7 +166,9 @@ def _split_list(lines):
         if in_fence:
             if cur is None:
                 cur = [i, "paragraph"]
-        elif _LIST_ITEM.match(line):
+        elif pymd and cur is not None and cur[1] == "paragraph" and not prev_blank:
+            pass                                   # Python-Markdown: no list without blank line
+        elif item_re.match(line):
             close(last_content + 1)
             cur = [i, "item"]
         elif _HR.match(line):

src/mdgrep/query.py

@@ -133,7 +133,11 @@ class Term:
     pattern: regex.Pattern = field(repr=False)
 
     def evaluate(self, text: str, deadline: float | None) -> bool:
+        # Words and phrases are escaped literals and run in linear time; only user-supplied
+        # regular expressions can backtrack catastrophically and need the (costly) timeout.
+        if self.kind == "regex":
             return self.pattern.search(text, timeout=_remaining(deadline)) is not None
+        return self.pattern.search(text) is not None
 
 
 @dataclass
@@ -285,6 +289,7 @@ class Query:
         if deadline is None and timeout is not None:
             deadline = time.monotonic() + timeout
         try:
+            _remaining(deadline)
             return self._root.evaluate(text, deadline)
         except TimeoutError as e:
             raise QueryTimeout(str(e) or "Search time limit exceeded") from None
@@ -296,7 +301,8 @@ class Query:
         found = []
         try:
             for term in self.positive_terms:
-                for m in term.pattern.finditer(text, timeout=_remaining(deadline)):
+                timeout = _remaining(deadline) if term.kind == "regex" else None
+                for m in term.pattern.finditer(text, timeout=timeout):
                     if m.end() > m.start():
                         found.append((m.start(), m.end()))
         except TimeoutError as e:

tests/test_markdown.py

@@ -63,6 +63,29 @@ def test_split_list():
     assert "not a heading" in blocks[6].text
 
 
+def test_dialect_list_interrupting_paragraph():
+    doc = "**1. Trend**\n* stories: a\n* why: b\n\n- real item\n"
+    common = split_markdown(doc, split="list")
+    assert [b.meta["kind"] for b in common] == ["paragraph", "item", "item", "item"]
+    pymd = split_markdown(doc, split="list", dialect="python-markdown")
+    assert [b.meta["kind"] for b in pymd] == ["paragraph", "item"]
+    assert "stories: a" in pymd[0].text
+
+
+def test_dialect_nesting_indent():
+    doc = "1. **Trend A**\n   - why: x\n   - how: y\n\n2. **Trend B**\n    - nested: z\n"
+    common = split_markdown(doc, split="list")
+    assert [b.line for b in common] == [1, 5]
+    pymd = split_markdown(doc, split="list", dialect="python-markdown")
+    assert [b.line for b in pymd] == [1, 2, 3, 5]      # 3 spaces: no nesting in Python-Markdown
+    assert "nested: z" in pymd[-1].text
+
+
+def test_invalid_dialect():
+    with pytest.raises(ValueError):
+        split_markdown("x", split="list", dialect="gfm")
+
+
 def test_split_heading():
     blocks = split_markdown(DOC, split="heading")
     assert [(b.title, b.anchor) for b in blocks] == [