dschlueter/mdgrep
Watch
1
0
Fork
You've already forked mdgrep
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: markdown dialect option, faster search (0.1.1)

Browse source 
- split_markdown/load_markdown: dialect="commonmark"|"python-markdown" so list
  blocks match the HTML structure of the renderer (list after paragraph needs a
  blank line; nesting needs 4 spaces in Python-Markdown)
- SearchEngine caches the searched text per block on load
- regex timeout only for /regex/ terms (words and phrases are linear);
  typical queries over 26k blocks: ~30-140 ms instead of 250-700 ms

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dschlueter 2026-10-12 19:37:40 +02:00
commit d9f81ac918
7 changed files with 85 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

8
README.md
View file

@ -12,8 +12,8 @@ Boolean full-text search over Markdown (and any other text) – as a **Python li
## Installation
```bash
pip install "mdgrep @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.0" # library + CLI
pip install "mdgrep[web] @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.0" # + Flask API
pip install "mdgrep @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.1" # library + CLI
pip install "mdgrep[web] @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.1" # + Flask API
```
Requires Python ≥ 3.10. Dependencies: [`regex`](https://pypi.org/project/regex/) (and `flask` for `[web]`).
@ -97,6 +97,10 @@ engine.search("login OR sso", filters={"status": "open"})
`load_markdown()` adds `meta["section"]` (nearest heading), `meta["kind"]`
(`file`/`section`/`item`/`paragraph`) and `meta["markdown"]` (raw source).
`dialect` (for `split="list"`) makes blocks line up with the HTML of your Markdown renderer:
`"commonmark"` (default) or `"python-markdown"` (a list needs a blank line before it; nesting
needs 4 spaces of indentation). Important if you link results to element ids in rendered HTML.
### `SearchEngine(source, *, reload_check=None, reload_interval=60)`
`source` is a list of blocks or a callable returning them. With a callable plus `reload_check`

2
pyproject.toml
View file

@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "mdgrep"
version = "0.1.0"
version = "0.1.1"
description = "Boolean full-text search (substrings, phrases, regex, AND/OR/NOT, parentheses) over Markdown and other text blocks, with CLI, Flask API and a JavaScript search widget"
readme = "README.md"
license = "MIT"

2
src/mdgrep/__init__.py
View file

@ -5,7 +5,7 @@ from .markdown import load_markdown, markdown_to_text, split_markdown
from .query import Query, QuerySyntaxError, QueryTimeout, compile_query
from .snippets import segments
__version__ = "0.1.0"
__version__ = "0.1.1"
STATIC_FILES = ("mdgrep-widget.js", "mdgrep-widget.css")

33
src/mdgrep/engine.py
View file

@ -103,12 +103,17 @@ class SearchEngine:
self._lock = threading.Lock()
self._token = reload_check() if reload_check else None
self._checked_at = time.monotonic()
self._blocks: list[Block] = list(source() if callable(source) else source)
self._set_blocks(list(source() if callable(source) else source))
def _set_blocks(self, blocks: list[Block]) -> None:
# Text that is searched (title + text), precomputed once per load.
haystacks = [f"{b.title}\n{b.text}" if b.title else b.text for b in blocks]
self._data = (blocks, haystacks)
@property
def blocks(self) -> list[Block]:
self._maybe_reload()
return self._blocks
return self._data[0]
def reload(self) -> None:
"""Reload blocks from the loader (no-op for a static block list)."""
@ -116,7 +121,7 @@ class SearchEngine:
return
blocks = list(self._loader())
with self._lock:
self._blocks = blocks
self._set_blocks(blocks)
self._token = self._reload_check() if self._reload_check else None
self._checked_at = time.monotonic()
@ -160,14 +165,19 @@ class SearchEngine:
started = time.monotonic()
q = query if isinstance(query, Query) else compile_query(query)
deadline = started + time_budget if time_budget else None
candidates = self._filter(self.blocks, filters, predicate)
self._maybe_reload()
blocks, haystacks = self._data
conditions = self._conditions(filters)
matched: list[Block] = []
truncated = False
for block in candidates:
for i, block in enumerate(blocks):
if conditions and any(block.meta.get(k) not in allowed for k, allowed in conditions):
continue
if predicate is not None and not predicate(block):
continue
try:
if q.matches(f"{block.title}\n{block.text}" if block.title else block.text,
deadline=deadline):
if q.matches(haystacks[i], deadline=deadline):
matched.append(block)
except QueryTimeout:
truncated = True
@ -214,16 +224,11 @@ class SearchEngine:
)
@staticmethod
def _filter(blocks, filters, predicate):
def _conditions(filters):
conditions = []
for key, value in (filters or {}).items():
if value is None or value == "" or value == []:
continue
allowed = set(value) if isinstance(value, (list, tuple, set, frozenset)) else {value}
conditions.append((key, allowed))
for block in blocks:
if any(block.meta.get(k) not in allowed for k, allowed in conditions):
continue
if predicate is not None and not predicate(block):
continue
yield block
return conditions

40
src/mdgrep/markdown.py
View file

@ -9,6 +9,13 @@ Split modes:
Every block gets `meta["section"]` (nearest preceding heading text), `meta["kind"]`
("file", "section", "item" or "paragraph") and `meta["markdown"]` (the raw source).
`dialect` (list mode) decides how list items are recognised, so blocks line up with the HTML
your Markdown renderer produces:
commonmark (default) a list may interrupt a paragraph; only unindented markers start
top-level items
python-markdown a list needs a blank line before it; markers indented by up to 3 spaces
still start top-level items (nesting needs 4 spaces)
The searchable `text` is plain text with Markdown syntax removed. In `file` and `heading`
mode the heading becomes the block title (and is removed from the text); list items and
paragraphs have no title.
@ -22,19 +29,22 @@ from typing import Callable, Iterable
from .engine import Block
__all__ = ["load_markdown", "split_markdown", "markdown_to_text", "slugify", "SPLIT_MODES"]
__all__ = ["load_markdown", "split_markdown", "markdown_to_text", "slugify", "SPLIT_MODES", "DIALECTS"]
SPLIT_MODES = ("file", "heading", "list", "paragraph")
DIALECTS = ("commonmark", "python-markdown")
_HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
_FENCE = re.compile(r"^\s*(```|~~~)")
_LIST_ITEM = re.compile(r"^([-*+]|\d{1,9}[.)])\s+")
_LIST_ITEM_PYMD = re.compile(r"^ {0,3}([-*+]|\d{1,9}\.)\s+")
_HR = re.compile(r"^\s*([-*_])(\s*\1){2,}\s*$")
def load_markdown(paths: str | Path | Iterable[str | Path], *, split: str = "paragraph",
meta: Callable[[Path], dict] | None = None, encoding: str = "utf-8",
pattern: str = "*.md", relative_to: str | Path | None = None) -> list[Block]:
pattern: str = "*.md", relative_to: str | Path | None = None,
dialect: str = "commonmark") -> list[Block]:
"""Load Markdown files (directories are searched recursively for `pattern`).
`meta(path)` may return a dict merged into every block's meta of that file.
@ -47,7 +57,7 @@ def load_markdown(paths: str | Path | Iterable[str | Path], *, split: str = "par
text = path.read_text(encoding=encoding)
extra = meta(path) if meta else {}
source = path.resolve().relative_to(base).as_posix() if base else str(path)
blocks.extend(split_markdown(text, source=source, split=split, meta=extra))
blocks.extend(split_markdown(text, source=source, split=split, meta=extra, dialect=dialect))
return blocks
@ -64,18 +74,19 @@ def _expand(paths, pattern):
def split_markdown(text: str, *, source: str = "", split: str = "paragraph",
meta: dict | None = None) -> list[Block]:
"""Split one Markdown document into blocks (see module docstring for modes)."""
meta: dict | None = None, dialect: str = "commonmark") -> list[Block]:
"""Split one Markdown document into blocks (see module docstring for modes and dialects)."""
if split not in SPLIT_MODES:
raise ValueError(f"split must be one of {SPLIT_MODES}, got {split!r}")
if dialect not in DIALECTS:
raise ValueError(f"dialect must be one of {DIALECTS}, got {dialect!r}")
meta = meta or {}
lines = text.splitlines()
raw_blocks = {
"file": _split_file,
"heading": _split_heading,
"list": _split_list,
"paragraph": _split_paragraph,
}[split](lines)
if split == "list":
raw_blocks = _split_list(lines, dialect == "python-markdown")
else:
raw_blocks = {"file": _split_file, "heading": _split_heading,
"paragraph": _split_paragraph}[split](lines)
blocks = []
for start, end, kind, section, title, anchor in raw_blocks:
@ -127,7 +138,7 @@ def _split_heading(lines):
return out
def _split_list(lines):
def _split_list(lines, pymd=False):
out = []
section = ""
cur = None # [start, kind]
@ -139,6 +150,7 @@ def _split_list(lines):
out.append((cur[0], end, cur[1], section, "", ""))
cur = None
item_re = _LIST_ITEM_PYMD if pymd else _LIST_ITEM
prev_blank = True
for i, line, in_fence, heading in _scan(lines):
if heading:
@ -154,7 +166,9 @@ def _split_list(lines):
if in_fence:
if cur is None:
cur = [i, "paragraph"]
elif _LIST_ITEM.match(line):
elif pymd and cur is not None and cur[1] == "paragraph" and not prev_blank:
pass # Python-Markdown: no list without blank line
elif item_re.match(line):
close(last_content + 1)
cur = [i, "item"]
elif _HR.match(line):

10
src/mdgrep/query.py
View file

@ -133,7 +133,11 @@ class Term:
pattern: regex.Pattern = field(repr=False)
def evaluate(self, text: str, deadline: float | None) -> bool:
return self.pattern.search(text, timeout=_remaining(deadline)) is not None
# Words and phrases are escaped literals and run in linear time; only user-supplied
# regular expressions can backtrack catastrophically and need the (costly) timeout.
if self.kind == "regex":
return self.pattern.search(text, timeout=_remaining(deadline)) is not None
return self.pattern.search(text) is not None
@dataclass
@ -285,6 +289,7 @@ class Query:
if deadline is None and timeout is not None:
deadline = time.monotonic() + timeout
try:
_remaining(deadline)
return self._root.evaluate(text, deadline)
except TimeoutError as e:
raise QueryTimeout(str(e) or "Search time limit exceeded") from None
@ -296,7 +301,8 @@ class Query:
found = []
try:
for term in self.positive_terms:
for m in term.pattern.finditer(text, timeout=_remaining(deadline)):
timeout = _remaining(deadline) if term.kind == "regex" else None
for m in term.pattern.finditer(text, timeout=timeout):
if m.end() > m.start():
found.append((m.start(), m.end()))
except TimeoutError as e:

23
tests/test_markdown.py
View file

@ -63,6 +63,29 @@ def test_split_list():
assert "not a heading" in blocks[6].text
def test_dialect_list_interrupting_paragraph():
doc = "**1. Trend**\n* stories: a\n* why: b\n\n- real item\n"
common = split_markdown(doc, split="list")
assert [b.meta["kind"] for b in common] == ["paragraph", "item", "item", "item"]
pymd = split_markdown(doc, split="list", dialect="python-markdown")
assert [b.meta["kind"] for b in pymd] == ["paragraph", "item"]
assert "stories: a" in pymd[0].text
def test_dialect_nesting_indent():
doc = "1. **Trend A**\n - why: x\n - how: y\n\n2. **Trend B**\n - nested: z\n"
common = split_markdown(doc, split="list")
assert [b.line for b in common] == [1, 5]
pymd = split_markdown(doc, split="list", dialect="python-markdown")
assert [b.line for b in pymd] == [1, 2, 3, 5] # 3 spaces: no nesting in Python-Markdown
assert "nested: z" in pymd[-1].text
def test_invalid_dialect():
with pytest.raises(ValueError):
split_markdown("x", split="list", dialect="gfm")
def test_split_heading():
blocks = split_markdown(DOC, split="heading")
assert [(b.title, b.anchor) for b in blocks] == [