diff --git a/README.md b/README.md index 7b43248..cdfacc5 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ Boolean full-text search over Markdown (and any other text) – as a **Python li ## Installation ```bash -pip install "mdgrep @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.0" # library + CLI -pip install "mdgrep[web] @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.0" # + Flask API +pip install "mdgrep @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.1" # library + CLI +pip install "mdgrep[web] @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.1" # + Flask API ``` Requires Python ≥ 3.10. Dependencies: [`regex`](https://pypi.org/project/regex/) (and `flask` for `[web]`). @@ -97,6 +97,10 @@ engine.search("login OR sso", filters={"status": "open"}) `load_markdown()` adds `meta["section"]` (nearest heading), `meta["kind"]` (`file`/`section`/`item`/`paragraph`) and `meta["markdown"]` (raw source). +`dialect` (for `split="list"`) makes blocks line up with the HTML of your Markdown renderer: +`"commonmark"` (default) or `"python-markdown"` (a list needs a blank line before it; nesting +needs 4 spaces of indentation). Important if you link results to element ids in rendered HTML. + ### `SearchEngine(source, *, reload_check=None, reload_interval=60)` `source` is a list of blocks or a callable returning them. With a callable plus `reload_check` diff --git a/pyproject.toml b/pyproject.toml index 71634c2..338948f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mdgrep" -version = "0.1.0" +version = "0.1.1" description = "Boolean full-text search (substrings, phrases, regex, AND/OR/NOT, parentheses) over Markdown and other text blocks, with CLI, Flask API and a JavaScript search widget" readme = "README.md" license = "MIT" diff --git a/src/mdgrep/__init__.py b/src/mdgrep/__init__.py index 5870ffb..6f54421 100644 --- a/src/mdgrep/__init__.py +++ b/src/mdgrep/__init__.py @@ -5,7 +5,7 @@ from .markdown import load_markdown, markdown_to_text, split_markdown from .query import Query, QuerySyntaxError, QueryTimeout, compile_query from .snippets import segments -__version__ = "0.1.0" +__version__ = "0.1.1" STATIC_FILES = ("mdgrep-widget.js", "mdgrep-widget.css") diff --git a/src/mdgrep/engine.py b/src/mdgrep/engine.py index db7ab79..69f4046 100644 --- a/src/mdgrep/engine.py +++ b/src/mdgrep/engine.py @@ -103,12 +103,17 @@ class SearchEngine: self._lock = threading.Lock() self._token = reload_check() if reload_check else None self._checked_at = time.monotonic() - self._blocks: list[Block] = list(source() if callable(source) else source) + self._set_blocks(list(source() if callable(source) else source)) + + def _set_blocks(self, blocks: list[Block]) -> None: + # Text that is searched (title + text), precomputed once per load. + haystacks = [f"{b.title}\n{b.text}" if b.title else b.text for b in blocks] + self._data = (blocks, haystacks) @property def blocks(self) -> list[Block]: self._maybe_reload() - return self._blocks + return self._data[0] def reload(self) -> None: """Reload blocks from the loader (no-op for a static block list).""" @@ -116,7 +121,7 @@ class SearchEngine: return blocks = list(self._loader()) with self._lock: - self._blocks = blocks + self._set_blocks(blocks) self._token = self._reload_check() if self._reload_check else None self._checked_at = time.monotonic() @@ -160,14 +165,19 @@ class SearchEngine: started = time.monotonic() q = query if isinstance(query, Query) else compile_query(query) deadline = started + time_budget if time_budget else None - candidates = self._filter(self.blocks, filters, predicate) + self._maybe_reload() + blocks, haystacks = self._data + conditions = self._conditions(filters) matched: list[Block] = [] truncated = False - for block in candidates: + for i, block in enumerate(blocks): + if conditions and any(block.meta.get(k) not in allowed for k, allowed in conditions): + continue + if predicate is not None and not predicate(block): + continue try: - if q.matches(f"{block.title}\n{block.text}" if block.title else block.text, - deadline=deadline): + if q.matches(haystacks[i], deadline=deadline): matched.append(block) except QueryTimeout: truncated = True @@ -214,16 +224,11 @@ class SearchEngine: ) @staticmethod - def _filter(blocks, filters, predicate): + def _conditions(filters): conditions = [] for key, value in (filters or {}).items(): if value is None or value == "" or value == []: continue allowed = set(value) if isinstance(value, (list, tuple, set, frozenset)) else {value} conditions.append((key, allowed)) - for block in blocks: - if any(block.meta.get(k) not in allowed for k, allowed in conditions): - continue - if predicate is not None and not predicate(block): - continue - yield block + return conditions diff --git a/src/mdgrep/markdown.py b/src/mdgrep/markdown.py index 6a647f4..07eef14 100644 --- a/src/mdgrep/markdown.py +++ b/src/mdgrep/markdown.py @@ -9,6 +9,13 @@ Split modes: Every block gets `meta["section"]` (nearest preceding heading text), `meta["kind"]` ("file", "section", "item" or "paragraph") and `meta["markdown"]` (the raw source). +`dialect` (list mode) decides how list items are recognised, so blocks line up with the HTML +your Markdown renderer produces: + commonmark (default) a list may interrupt a paragraph; only unindented markers start + top-level items + python-markdown a list needs a blank line before it; markers indented by up to 3 spaces + still start top-level items (nesting needs 4 spaces) + The searchable `text` is plain text with Markdown syntax removed. In `file` and `heading` mode the heading becomes the block title (and is removed from the text); list items and paragraphs have no title. @@ -22,19 +29,22 @@ from typing import Callable, Iterable from .engine import Block -__all__ = ["load_markdown", "split_markdown", "markdown_to_text", "slugify", "SPLIT_MODES"] +__all__ = ["load_markdown", "split_markdown", "markdown_to_text", "slugify", "SPLIT_MODES", "DIALECTS"] SPLIT_MODES = ("file", "heading", "list", "paragraph") +DIALECTS = ("commonmark", "python-markdown") _HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$") _FENCE = re.compile(r"^\s*(```|~~~)") _LIST_ITEM = re.compile(r"^([-*+]|\d{1,9}[.)])\s+") +_LIST_ITEM_PYMD = re.compile(r"^ {0,3}([-*+]|\d{1,9}\.)\s+") _HR = re.compile(r"^\s*([-*_])(\s*\1){2,}\s*$") def load_markdown(paths: str | Path | Iterable[str | Path], *, split: str = "paragraph", meta: Callable[[Path], dict] | None = None, encoding: str = "utf-8", - pattern: str = "*.md", relative_to: str | Path | None = None) -> list[Block]: + pattern: str = "*.md", relative_to: str | Path | None = None, + dialect: str = "commonmark") -> list[Block]: """Load Markdown files (directories are searched recursively for `pattern`). `meta(path)` may return a dict merged into every block's meta of that file. @@ -47,7 +57,7 @@ def load_markdown(paths: str | Path | Iterable[str | Path], *, split: str = "par text = path.read_text(encoding=encoding) extra = meta(path) if meta else {} source = path.resolve().relative_to(base).as_posix() if base else str(path) - blocks.extend(split_markdown(text, source=source, split=split, meta=extra)) + blocks.extend(split_markdown(text, source=source, split=split, meta=extra, dialect=dialect)) return blocks @@ -64,18 +74,19 @@ def _expand(paths, pattern): def split_markdown(text: str, *, source: str = "", split: str = "paragraph", - meta: dict | None = None) -> list[Block]: - """Split one Markdown document into blocks (see module docstring for modes).""" + meta: dict | None = None, dialect: str = "commonmark") -> list[Block]: + """Split one Markdown document into blocks (see module docstring for modes and dialects).""" if split not in SPLIT_MODES: raise ValueError(f"split must be one of {SPLIT_MODES}, got {split!r}") + if dialect not in DIALECTS: + raise ValueError(f"dialect must be one of {DIALECTS}, got {dialect!r}") meta = meta or {} lines = text.splitlines() - raw_blocks = { - "file": _split_file, - "heading": _split_heading, - "list": _split_list, - "paragraph": _split_paragraph, - }[split](lines) + if split == "list": + raw_blocks = _split_list(lines, dialect == "python-markdown") + else: + raw_blocks = {"file": _split_file, "heading": _split_heading, + "paragraph": _split_paragraph}[split](lines) blocks = [] for start, end, kind, section, title, anchor in raw_blocks: @@ -127,7 +138,7 @@ def _split_heading(lines): return out -def _split_list(lines): +def _split_list(lines, pymd=False): out = [] section = "" cur = None # [start, kind] @@ -139,6 +150,7 @@ def _split_list(lines): out.append((cur[0], end, cur[1], section, "", "")) cur = None + item_re = _LIST_ITEM_PYMD if pymd else _LIST_ITEM prev_blank = True for i, line, in_fence, heading in _scan(lines): if heading: @@ -154,7 +166,9 @@ def _split_list(lines): if in_fence: if cur is None: cur = [i, "paragraph"] - elif _LIST_ITEM.match(line): + elif pymd and cur is not None and cur[1] == "paragraph" and not prev_blank: + pass # Python-Markdown: no list without blank line + elif item_re.match(line): close(last_content + 1) cur = [i, "item"] elif _HR.match(line): diff --git a/src/mdgrep/query.py b/src/mdgrep/query.py index 9749581..c89deb3 100644 --- a/src/mdgrep/query.py +++ b/src/mdgrep/query.py @@ -133,7 +133,11 @@ class Term: pattern: regex.Pattern = field(repr=False) def evaluate(self, text: str, deadline: float | None) -> bool: - return self.pattern.search(text, timeout=_remaining(deadline)) is not None + # Words and phrases are escaped literals and run in linear time; only user-supplied + # regular expressions can backtrack catastrophically and need the (costly) timeout. + if self.kind == "regex": + return self.pattern.search(text, timeout=_remaining(deadline)) is not None + return self.pattern.search(text) is not None @dataclass @@ -285,6 +289,7 @@ class Query: if deadline is None and timeout is not None: deadline = time.monotonic() + timeout try: + _remaining(deadline) return self._root.evaluate(text, deadline) except TimeoutError as e: raise QueryTimeout(str(e) or "Search time limit exceeded") from None @@ -296,7 +301,8 @@ class Query: found = [] try: for term in self.positive_terms: - for m in term.pattern.finditer(text, timeout=_remaining(deadline)): + timeout = _remaining(deadline) if term.kind == "regex" else None + for m in term.pattern.finditer(text, timeout=timeout): if m.end() > m.start(): found.append((m.start(), m.end())) except TimeoutError as e: diff --git a/tests/test_markdown.py b/tests/test_markdown.py index b4d3bb1..48a21ef 100644 --- a/tests/test_markdown.py +++ b/tests/test_markdown.py @@ -63,6 +63,29 @@ def test_split_list(): assert "not a heading" in blocks[6].text +def test_dialect_list_interrupting_paragraph(): + doc = "**1. Trend**\n* stories: a\n* why: b\n\n- real item\n" + common = split_markdown(doc, split="list") + assert [b.meta["kind"] for b in common] == ["paragraph", "item", "item", "item"] + pymd = split_markdown(doc, split="list", dialect="python-markdown") + assert [b.meta["kind"] for b in pymd] == ["paragraph", "item"] + assert "stories: a" in pymd[0].text + + +def test_dialect_nesting_indent(): + doc = "1. **Trend A**\n - why: x\n - how: y\n\n2. **Trend B**\n - nested: z\n" + common = split_markdown(doc, split="list") + assert [b.line for b in common] == [1, 5] + pymd = split_markdown(doc, split="list", dialect="python-markdown") + assert [b.line for b in pymd] == [1, 2, 3, 5] # 3 spaces: no nesting in Python-Markdown + assert "nested: z" in pymd[-1].text + + +def test_invalid_dialect(): + with pytest.raises(ValueError): + split_markdown("x", split="list", dialect="gfm") + + def test_split_heading(): blocks = split_markdown(DOC, split="heading") assert [(b.title, b.anchor) for b in blocks] == [