commit 9c7b7e81f96d12c41277d390ae50f6ba689d9637 Author: dschlueter Date: Mon Oct 12 19:29:44 2026 +0200 feat: mdgrep 0.1.0 – boolean full-text search for Markdown - query language: substrings, phrases, /regex/, AND/OR/NOT (+ UND/ODER/NICHT), parentheses; case-insensitive; ReDoS protection via regex timeouts; syntax errors with stable code and position - SearchEngine over in-memory blocks: filters, predicate, grouping, sorting, paging, time budget, automatic reload, highlighted snippets - Markdown loader with split modes file/heading/list/paragraph - grep-like CLI (mdgrep), JSON output, grep exit codes - Flask blueprint/app with JSON API and widget assets, dev server - framework-free JS widget: live search, URL state, filters, paging, syntax help, en/de labels, CSS custom properties for theming Co-Authored-By: Claude Opus 5.5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f6412bd --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.venv/ +__pycache__/ +*.egg-info/ +dist/ +build/ +.pytest_cache/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0c1ff7f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Dieter Schlüter + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7b43248 --- /dev/null +++ b/README.md @@ -0,0 +1,220 @@ +# mdgrep + +Boolean full-text search over Markdown (and any other text) – as a **Python library**, a +**grep-like command line tool**, a **Flask JSON API** and a **framework-free JavaScript search widget**. + +- substrings, `"phrases"` and `/regular expressions/`, always case-insensitive (Unicode-aware) +- `AND` / `OR` / `NOT` with parentheses – German keywords `UND` / `ODER` / `NICHT` work too +- results as "boxes" (blocks) with highlighted context, grouping, sorting, filters and paging +- protection against catastrophic regular expressions (time limit per search) +- no search index: blocks live in memory, a few MB of text are searched in milliseconds + +## Installation + +```bash +pip install "mdgrep @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.0" # library + CLI +pip install "mdgrep[web] @ git+https://kitux.de/forgejo/dschlueter/mdgrep.git@v0.1.0" # + Flask API +``` + +Requires Python ≥ 3.10. Dependencies: [`regex`](https://pypi.org/project/regex/) (and `flask` for `[web]`). + +## Query syntax + +| Query | Meaning | +|---|---| +| `rust` | substring, case-insensitive – also matches *Rust*, *trusted*, *Rustacean* | +| `"open source"` | phrase; any whitespace (incl. line breaks) between the words | +| `/llms?\b/` | regular expression (`\/` for a literal slash), case-insensitive | +| `a b` · `a AND b` · `a UND b` · `a & b` | both | +| `a OR b` · `a ODER b` · `a \| b` | at least one | +| `NOT a` · `NICHT a` · `-a` · `!a` | must not occur | +| `( … )` | grouping | + +Precedence: `NOT` > `AND` > `OR`, so `a OR b AND c` means `a OR (b AND c)`. +Keywords are only recognised in upper case (`and` is a normal search word). +Whole words: use a regex, e.g. `/\bgo\b/`. + +Examples: `(rust OR go) AND NOT crypto` · `"machine learning" -/pytorch|tensorflow/` · `llm (privacy OR datenschutz)` + +Invalid queries raise `QuerySyntaxError` with a stable `code`, a `message` and the 0-based `position`: +`empty_query`, `empty_phrase`, `unterminated_phrase`, `unterminated_regex`, `invalid_regex`, +`missing_operand`, `unexpected_paren`, `unexpected_token`, `unclosed_paren`, `empty_group`, +`too_long`, `too_many_terms`. + +## Command line + +```bash +mdgrep '(rust OR go) AND NOT crypto' notes/ # search all *.md below notes/ +mdgrep '/llms?\b/' docs/*.md --split list # one result per list item +mdgrep 'todo' . -l # file names only +mdgrep 'release' . -c # number of matching blocks +mdgrep '"open source"' . --json # JSON (same format as the Web API) +python -m mdgrep --help +``` + +`--split` chooses the unit that is searched and shown: `paragraph` (default), `list` +(top-level list items, other text as paragraphs), `heading` (sections) or `file`. +Exit codes as with grep: `0` match, `1` no match, `2` error (syntax error, time limit). + +## Python API + +```python +from mdgrep import compile_query, load_markdown, SearchEngine, Block + +# 1. Just the query language – on any string +q = compile_query('(rust OR go) AND NOT /crypto(currency)?/') +q.matches("Rust 2.0 is out") # True +q.spans("Rust and Go") # [(0, 4), (9, 11)] – for highlighting + +# 2. Markdown files → blocks → search +blocks = load_markdown("docs/", split="list", relative_to="docs/", + meta=lambda path: {"lang": path.stem.rsplit("_", 1)[-1]}) +engine = SearchEngine(blocks) +result = engine.search("rust", filters={"lang": "de"}, limit=20) +for group in result.groups: + print(group.block.source, group.block.line, group.segments) +result.to_dict() # JSON-ready, see "JSON format" + +# 3. Your own data – any text source +engine = SearchEngine([ + Block(id="42", title="Ticket 42", text="Login fails with SSO", meta={"status": "open"}), + Block(id="43", title="Ticket 43", text="Dark mode toggle broken", meta={"status": "closed"}), +]) +engine.search("login OR sso", filters={"status": "open"}) +``` + +### `Block` + +| Field | Meaning | +|---|---| +| `id` | unique id | +| `text` | searched and shown as snippet | +| `title` | searched and shown as heading (don't repeat it in `text`) | +| `anchor` | optional fragment id, used by the widget as link target if no `href` is given | +| `source`, `line` | origin (file, 1-based line) | +| `meta` | free dict, passed through unchanged (filters work on it) | + +`load_markdown()` adds `meta["section"]` (nearest heading), `meta["kind"]` +(`file`/`section`/`item`/`paragraph`) and `meta["markdown"]` (raw source). + +### `SearchEngine(source, *, reload_check=None, reload_interval=60)` + +`source` is a list of blocks or a callable returning them. With a callable plus `reload_check` +(returns any comparable token, e.g. newest file mtime) blocks are reloaded automatically when +the token changes – checked at most every `reload_interval` seconds: + +```python +engine = SearchEngine(lambda: load_markdown("docs/"), + reload_check=lambda: max(p.stat().st_mtime for p in Path("docs").rglob("*.md"))) +``` + +### `engine.search(query, **options)` + +| Option | Default | Meaning | +|---|---|---| +| `filters` | `None` | `{meta_key: value or [values]}`, all keys must match; empty values are ignored | +| `predicate` | `None` | extra `callable(block) -> bool` | +| `group_by` | `None` | `callable(block) -> key or None`; one result per key, the others in `also` | +| `sort_key`, `reverse` | original order | sort matching blocks (e.g. newest first) | +| `offset`, `limit` | `0`, `30` | paging, counted in groups (`limit=None` = all) | +| `time_budget` | `1.5` | seconds for the whole search; on overrun `truncated=True` | +| `snippet_chars`, `context` | `700`, `120` | longer texts are cut to windows around the hits | + +## Web API (Flask) + +```python +from mdgrep import SearchEngine, load_markdown +from mdgrep.web import create_app, create_blueprint + +engine = SearchEngine(lambda: load_markdown("docs", split="list", relative_to="docs")) +app = create_app(engine, filters=["lang"]) # standalone: GET /search +# or inside an existing app: +app.register_blueprint(create_blueprint(engine, filters=["lang"]), url_prefix="/api") +``` + +`GET /search?q=…&offset=0&limit=30&=value` – for each name in `filters` an equally +named parameter (`""`/`all` = no restriction, `a,b` = several values). The blueprint also serves the +widget assets under `/mdgrep-static/`. `create_blueprint()` accepts the search options above plus +`default_limit`, `max_limit`, `max_query_length` and `hide_meta` (meta keys removed from responses, +default `("markdown",)`). + +Development server (serves a static site and the API together): + +```bash +python -m mdgrep.web --dir examples/docs --split list --static examples/site --port 8000 +# → http://127.0.0.1:8000/ (API: /api/search) +``` + +For production run the app with a WSGI server (e.g. `gunicorn -w 2 module:app`) behind a reverse +proxy with rate limiting. + +### JSON format + +```json +{ + "query": "rust", "total_groups": 12, "total_blocks": 15, + "offset": 0, "limit": 30, "next_offset": null, "truncated": false, "elapsed_ms": 3.2, + "results": [{ + "block": { + "id": "a.md:4", "title": "…", "anchor": "", "source": "a.md", "line": 4, "meta": {}, + "title_segments": [{"t": "…", "hit": false}], + "segments": [{"t": "The ", "hit": false}, {"t": "Rust", "hit": true}, {"t": " team …", "hit": false}], + "hits": 1 + }, + "also": [{"id": "b.md:9", "title": "…", "anchor": "", "source": "b.md", "line": 9, "meta": {}}] + }] +} +``` + +Errors: HTTP 400 `{"error": {"code": "unclosed_paren", "message": "…", "position": 0}}`. + +## JavaScript widget + +Framework-free, live search with debounce, cancels outdated requests, keeps the query in the URL +(shareable, back button), filters, "load more", collapsible syntax help, error messages with +position, English and German labels. All result text is inserted via `textContent` (no XSS). + +```html + + + + +``` + +Without JavaScript code: `
`. + +Other options: `live`, `debounce` (300 ms), `minChars` (2), `limit` (30), `syncUrl`, `autofocus`, +`help`, `href(block)`, `formatBadge(field, value, block)`, `alsoText(block)`. +The widget fires a `mdgrep:results` event with the response on its root element. + +For static sites copy the assets instead of serving them from Flask: + +```python +import mdgrep +mdgrep.export_static("site/static") # writes mdgrep-widget.js / .css +``` + +Theming via CSS custom properties (defaults are light): +`--mdgrep-text`, `--mdgrep-muted`, `--mdgrep-surface`, `--mdgrep-border`, `--mdgrep-accent`, +`--mdgrep-mark-bg`, `--mdgrep-mark-text`, `--mdgrep-error`, `--mdgrep-radius`. + +## Development + +```bash +python -m venv .venv && .venv/bin/pip install -e ".[dev]" +.venv/bin/pytest +``` + +## License + +MIT diff --git a/examples/docs/languages.md b/examples/docs/languages.md new file mode 100644 index 0000000..629477b --- /dev/null +++ b/examples/docs/languages.md @@ -0,0 +1,13 @@ +# Programming languages + +## Systems + +- **Rust** guarantees memory safety without a garbage collector. Its borrow checker + rejects data races at compile time. +- **Go** favours simplicity and fast compilation; goroutines make concurrency cheap. +- **Zig** offers manual memory management with compile-time code execution. + +## Scripting + +- **Python** is popular for data science and machine learning (LLMs, PyTorch). +- **Lua** is a tiny embeddable language used in games and Neovim. diff --git a/examples/docs/tools.md b/examples/docs/tools.md new file mode 100644 index 0000000..a5fe579 --- /dev/null +++ b/examples/docs/tools.md @@ -0,0 +1,7 @@ +# Tools + +Regular expressions are supported by grep, ripgrep and most editors. + +Large language models (LLMs) can explain a regex, but testing it is still wise. + +Open source licenses such as MIT and GPL differ in their copyleft requirements. diff --git a/examples/site/index.html b/examples/site/index.html new file mode 100644 index 0000000..e255c0e --- /dev/null +++ b/examples/site/index.html @@ -0,0 +1,21 @@ + + + + + + mdgrep demo + + + + +

mdgrep demo

+ + + + + + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..71634c2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "mdgrep" +version = "0.1.0" +description = "Boolean full-text search (substrings, phrases, regex, AND/OR/NOT, parentheses) over Markdown and other text blocks, with CLI, Flask API and a JavaScript search widget" +readme = "README.md" +license = "MIT" +requires-python = ">=3.10" +authors = [{ name = "Dieter Schlüter" }] +dependencies = ["regex>=2023.0"] + +[project.optional-dependencies] +web = ["flask>=2.3"] +dev = ["pytest>=7", "flask>=2.3"] + +[project.scripts] +mdgrep = "mdgrep.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +mdgrep = ["static/*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/src/mdgrep/__init__.py b/src/mdgrep/__init__.py new file mode 100644 index 0000000..5870ffb --- /dev/null +++ b/src/mdgrep/__init__.py @@ -0,0 +1,47 @@ +"""mdgrep – boolean full-text search over Markdown and other text blocks.""" + +from .engine import Block, Group, SearchEngine, SearchResult +from .markdown import load_markdown, markdown_to_text, split_markdown +from .query import Query, QuerySyntaxError, QueryTimeout, compile_query +from .snippets import segments + +__version__ = "0.1.0" + +STATIC_FILES = ("mdgrep-widget.js", "mdgrep-widget.css") + + +def export_static(target_dir) -> list: + """Copy the widget assets (JS + CSS) into `target_dir`, e.g. a static site's asset folder. + + Returns the written paths. Files are only rewritten when their content changed. + """ + from importlib.resources import files + from pathlib import Path + + target = Path(target_dir) + target.mkdir(parents=True, exist_ok=True) + written = [] + for name in STATIC_FILES: + data = files("mdgrep").joinpath("static", name).read_bytes() + dest = target / name + if not dest.exists() or dest.read_bytes() != data: + dest.write_bytes(data) + written.append(dest) + return written + +__all__ = [ + "Block", + "Group", + "Query", + "QuerySyntaxError", + "QueryTimeout", + "SearchEngine", + "SearchResult", + "compile_query", + "export_static", + "load_markdown", + "markdown_to_text", + "split_markdown", + "segments", + "__version__", +] diff --git a/src/mdgrep/__main__.py b/src/mdgrep/__main__.py new file mode 100644 index 0000000..dd8a8c9 --- /dev/null +++ b/src/mdgrep/__main__.py @@ -0,0 +1,5 @@ +import sys + +from .cli import main + +sys.exit(main()) diff --git a/src/mdgrep/cli.py b/src/mdgrep/cli.py new file mode 100644 index 0000000..23deccf --- /dev/null +++ b/src/mdgrep/cli.py @@ -0,0 +1,123 @@ +"""grep-like command line: mdgrep QUERY [PATH ...] + +Exit codes as with grep: 0 = matches found, 1 = no match, 2 = error. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys + +from . import __version__ +from .engine import SearchEngine +from .markdown import SPLIT_MODES, load_markdown +from .query import QuerySyntaxError, compile_query + +_SYNTAX = """query syntax (case-insensitive): + word substring ("rust" matches "Rustacean") + "a phrase" exact phrase + /regex/ regular expression (\\/ for a literal slash) + a b, a AND b both (also UND, &) + a OR b either (also ODER, |) + NOT a, -a, !a negation (also NICHT) + ( ... ) grouping; precedence NOT > AND > OR + +examples: + mdgrep '(rust OR go) AND NOT crypto' notes/ + mdgrep '/llms?\\b/ "open source"' --split list docs/*.md +""" + +_COLORS = {"file": "\033[35m", "line": "\033[32m", "hit": "\033[1;31m", "dim": "\033[2m", "reset": "\033[0m"} + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="mdgrep", + description="Boolean full-text search in Markdown files.", + epilog=_SYNTAX, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("query", help="search query (see syntax below)") + p.add_argument("paths", nargs="*", default=["."], help="files or directories (default: .)") + p.add_argument("-s", "--split", choices=SPLIT_MODES, default="paragraph", + help="unit that is searched and shown (default: paragraph)") + p.add_argument("-g", "--glob", default="*.md", help="file pattern in directories (default: *.md)") + p.add_argument("-c", "--count", action="store_true", help="only print the number of matching blocks") + p.add_argument("-l", "--files-with-matches", action="store_true", help="only print file names") + p.add_argument("-j", "--json", action="store_true", help="JSON output (same format as the Web API)") + p.add_argument("-C", "--context", type=int, default=80, metavar="N", + help="characters of context around hits in long blocks (default: 80)") + p.add_argument("--max-chars", type=int, default=400, metavar="N", + help="shorten blocks longer than N characters (0 = never, default: 400)") + p.add_argument("-m", "--max-count", type=int, default=0, metavar="N", + help="stop after N matching blocks (0 = all)") + p.add_argument("--timeout", type=float, default=30.0, help="time limit in seconds (default: 30)") + p.add_argument("--color", choices=("auto", "always", "never"), default="auto") + p.add_argument("-V", "--version", action="version", version=f"mdgrep {__version__}") + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + color = args.color == "always" or (args.color == "auto" and sys.stdout.isatty() + and os.environ.get("NO_COLOR") is None) + c = _COLORS if color else dict.fromkeys(_COLORS, "") + + try: + query = compile_query(args.query) + except QuerySyntaxError as e: + print(f"mdgrep: {e.message}", file=sys.stderr) + print(f" {args.query}\n {' ' * e.position}^", file=sys.stderr) + return 2 + + try: + blocks = load_markdown(args.paths, split=args.split, pattern=args.glob) + except OSError as e: + print(f"mdgrep: {e}", file=sys.stderr) + return 2 + + result = SearchEngine(blocks).search( + query, + limit=args.max_count or None, + time_budget=args.timeout, + snippet_chars=args.max_chars or None, + context=args.context, + ) + if result.truncated: + print("mdgrep: time limit exceeded, results are incomplete", file=sys.stderr) + + if args.json: + data = result.to_dict() + for r in data["results"]: + r["block"]["meta"].pop("markdown", None) + json.dump(data, sys.stdout, ensure_ascii=False, indent=2) + print() + elif args.count: + print(result.total_blocks) + elif args.files_with_matches: + seen = [] + for g in result.groups: + if g.block.source not in seen: + seen.append(g.block.source) + print(f"{c['file']}{g.block.source}{c['reset']}") + else: + for g in result.groups: + print(f"{c['file']}{g.block.source}{c['reset']}:{c['line']}{g.block.line}{c['reset']}") + if g.block.title: + title = "".join(f"{c['hit']}{s['t']}{c['reset']}" if s["hit"] else s["t"] + for s in g.title_segments) + print(f" {title}") + text = "".join(f"{c['hit']}{s['t']}{c['reset']}" if s["hit"] else s["t"] for s in g.segments) + for line in text.split("\n"): + print(f" {line}" if line else "") + print() + + if result.truncated: + return 2 + return 0 if result.total_blocks else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/mdgrep/engine.py b/src/mdgrep/engine.py new file mode 100644 index 0000000..db7ab79 --- /dev/null +++ b/src/mdgrep/engine.py @@ -0,0 +1,229 @@ +"""Search engine over in-memory text blocks: filtering, grouping, sorting, paging, time budget.""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Hashable, Iterable + +from .query import Query, QueryTimeout, compile_query +from .snippets import segments + +__all__ = ["Block", "SearchEngine", "SearchResult", "Group"] + + +@dataclass +class Block: + """One searchable unit ("box" in the result list). + + `title` and `text` are both searched and shown (title as heading, text as snippet), so + loaders should not repeat the title inside the text. `meta` holds arbitrary application + data (e.g. language, date, URL) and is passed through to results unchanged. + """ + id: str + text: str + title: str = "" + anchor: str = "" + source: str = "" + line: int = 0 + meta: dict[str, Any] = field(default_factory=dict) + + def summary(self) -> dict: + return {"id": self.id, "title": self.title, "anchor": self.anchor, + "source": self.source, "line": self.line, "meta": self.meta} + + +@dataclass +class Group: + """A primary block with its highlighted snippet plus further blocks of the same group.""" + key: Hashable + block: Block + title_segments: list[dict] + segments: list[dict] + hits: int + also: list[Block] + + def to_dict(self) -> dict: + data = self.block.summary() + data["title_segments"] = self.title_segments + data["segments"] = self.segments + data["hits"] = self.hits + return {"block": data, "also": [b.summary() for b in self.also]} + + +@dataclass +class SearchResult: + query: str + groups: list[Group] + total_groups: int + total_blocks: int + offset: int + limit: int + truncated: bool + elapsed_ms: float + + @property + def next_offset(self) -> int | None: + nxt = self.offset + len(self.groups) + return nxt if nxt < self.total_groups else None + + def to_dict(self) -> dict: + """JSON contract shared by the Web API, the CLI (--json) and the widget.""" + return { + "query": self.query, + "total_groups": self.total_groups, + "total_blocks": self.total_blocks, + "offset": self.offset, + "limit": self.limit, + "next_offset": self.next_offset, + "truncated": self.truncated, + "elapsed_ms": round(self.elapsed_ms, 1), + "results": [g.to_dict() for g in self.groups], + } + + +BlockSource = Iterable[Block] | Callable[[], Iterable[Block]] + + +class SearchEngine: + """Holds blocks in memory and searches them. + + `source` is either an iterable of blocks or a callable returning one. With a callable and + `reload_check` (a callable returning any comparable token, e.g. the newest file mtime), + the blocks are reloaded automatically when the token changes; the check runs at most + every `reload_interval` seconds. + """ + + def __init__(self, source: BlockSource, *, reload_check: Callable[[], Any] | None = None, + reload_interval: float = 60.0): + self._loader = source if callable(source) else None + self._reload_check = reload_check + self._reload_interval = reload_interval + self._lock = threading.Lock() + self._token = reload_check() if reload_check else None + self._checked_at = time.monotonic() + self._blocks: list[Block] = list(source() if callable(source) else source) + + @property + def blocks(self) -> list[Block]: + self._maybe_reload() + return self._blocks + + def reload(self) -> None: + """Reload blocks from the loader (no-op for a static block list).""" + if self._loader is None: + return + blocks = list(self._loader()) + with self._lock: + self._blocks = blocks + self._token = self._reload_check() if self._reload_check else None + self._checked_at = time.monotonic() + + def _maybe_reload(self) -> None: + if self._loader is None or self._reload_check is None: + return + if time.monotonic() - self._checked_at < self._reload_interval: + return + self._checked_at = time.monotonic() + token = self._reload_check() + if token != self._token: + self.reload() + + def search( + self, + query: str | Query, + *, + filters: dict[str, Any] | None = None, + predicate: Callable[[Block], bool] | None = None, + group_by: Callable[[Block], Hashable | None] | None = None, + sort_key: Callable[[Block], Any] | None = None, + reverse: bool = False, + offset: int = 0, + limit: int | None = 30, + time_budget: float | None = 1.5, + snippet_chars: int | None = 700, + context: int = 120, + ) -> SearchResult: + """Search all blocks. + + filters {meta_key: value or list of values}; blocks must match all keys. + predicate extra filter function on blocks. + group_by returns a group key per block (None = block forms its own group). Each group + is shown once; its first block after sorting is the primary one. + sort_key sort matching blocks (default: original order); `reverse` for descending. + limit groups per page (None = all). `offset` counts groups. + time_budget seconds for the whole search; on overrun the result is partial and + `truncated` is True. + snippet_chars shorten long texts to windows around the hits (None = full text). + """ + started = time.monotonic() + q = query if isinstance(query, Query) else compile_query(query) + deadline = started + time_budget if time_budget else None + candidates = self._filter(self.blocks, filters, predicate) + + matched: list[Block] = [] + truncated = False + for block in candidates: + try: + if q.matches(f"{block.title}\n{block.text}" if block.title else block.text, + deadline=deadline): + matched.append(block) + except QueryTimeout: + truncated = True + break + + if sort_key is not None: + matched.sort(key=sort_key, reverse=reverse) + elif reverse: + matched.reverse() + + grouped: dict[Hashable, list[Block]] = {} + for i, block in enumerate(matched): + key = group_by(block) if group_by else None + grouped.setdefault(key if key is not None else ("__block__", i), []).append(block) + group_items = list(grouped.items()) + + page = group_items[offset:offset + limit] if limit is not None else group_items[offset:] + groups = [] + for key, members in page: + primary = members[0] + try: + spans = q.spans(primary.text, deadline=deadline) + title_spans = q.spans(primary.title, deadline=deadline) if primary.title else [] + except QueryTimeout: + spans, title_spans, truncated = [], [], True + groups.append(Group( + key=key, + block=primary, + title_segments=segments(primary.title, title_spans), + segments=segments(primary.text, spans, max_chars=snippet_chars, context=context), + hits=len(spans), + also=members[1:], + )) + + return SearchResult( + query=q.text, + groups=groups, + total_groups=len(group_items), + total_blocks=len(matched), + offset=offset, + limit=limit if limit is not None else len(group_items), + truncated=truncated, + elapsed_ms=(time.monotonic() - started) * 1000, + ) + + @staticmethod + def _filter(blocks, filters, predicate): + conditions = [] + for key, value in (filters or {}).items(): + if value is None or value == "" or value == []: + continue + allowed = set(value) if isinstance(value, (list, tuple, set, frozenset)) else {value} + conditions.append((key, allowed)) + for block in blocks: + if any(block.meta.get(k) not in allowed for k, allowed in conditions): + continue + if predicate is not None and not predicate(block): + continue + yield block diff --git a/src/mdgrep/markdown.py b/src/mdgrep/markdown.py new file mode 100644 index 0000000..6a647f4 --- /dev/null +++ b/src/mdgrep/markdown.py @@ -0,0 +1,240 @@ +"""Split Markdown files into searchable blocks. + +Split modes: + file one block per file + heading one block per section (heading + content up to the next heading) + list one block per top-level list item (with its indented continuation lines); + text outside lists becomes paragraph blocks + paragraph one block per blank-line separated paragraph (list items count as paragraphs) + +Every block gets `meta["section"]` (nearest preceding heading text), `meta["kind"]` +("file", "section", "item" or "paragraph") and `meta["markdown"]` (the raw source). +The searchable `text` is plain text with Markdown syntax removed. In `file` and `heading` +mode the heading becomes the block title (and is removed from the text); list items and +paragraphs have no title. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Callable, Iterable + +from .engine import Block + +__all__ = ["load_markdown", "split_markdown", "markdown_to_text", "slugify", "SPLIT_MODES"] + +SPLIT_MODES = ("file", "heading", "list", "paragraph") + +_HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$") +_FENCE = re.compile(r"^\s*(```|~~~)") +_LIST_ITEM = re.compile(r"^([-*+]|\d{1,9}[.)])\s+") +_HR = re.compile(r"^\s*([-*_])(\s*\1){2,}\s*$") + + +def load_markdown(paths: str | Path | Iterable[str | Path], *, split: str = "paragraph", + meta: Callable[[Path], dict] | None = None, encoding: str = "utf-8", + pattern: str = "*.md", relative_to: str | Path | None = None) -> list[Block]: + """Load Markdown files (directories are searched recursively for `pattern`). + + `meta(path)` may return a dict merged into every block's meta of that file. + `relative_to` makes `Block.source`/`Block.id` relative to that directory (recommended for + web use, so no server paths are exposed). + """ + base = Path(relative_to).resolve() if relative_to is not None else None + blocks: list[Block] = [] + for path in _expand(paths, pattern): + text = path.read_text(encoding=encoding) + extra = meta(path) if meta else {} + source = path.resolve().relative_to(base).as_posix() if base else str(path) + blocks.extend(split_markdown(text, source=source, split=split, meta=extra)) + return blocks + + +def _expand(paths, pattern): + if isinstance(paths, (str, Path)): + paths = [paths] + files = [] + for p in map(Path, paths): + if p.is_dir(): + files.extend(sorted(p.rglob(pattern))) + else: + files.append(p) + return files + + +def split_markdown(text: str, *, source: str = "", split: str = "paragraph", + meta: dict | None = None) -> list[Block]: + """Split one Markdown document into blocks (see module docstring for modes).""" + if split not in SPLIT_MODES: + raise ValueError(f"split must be one of {SPLIT_MODES}, got {split!r}") + meta = meta or {} + lines = text.splitlines() + raw_blocks = { + "file": _split_file, + "heading": _split_heading, + "list": _split_list, + "paragraph": _split_paragraph, + }[split](lines) + + blocks = [] + for start, end, kind, section, title, anchor in raw_blocks: + raw = "\n".join(lines[start:end]).strip("\n") + plain = markdown_to_text(raw) + if title and plain.startswith(title): + plain = plain[len(title):].lstrip("\n") + if not plain.strip() and not title: + continue + block_meta = {**meta, "section": section, "kind": kind, "markdown": raw} + blocks.append(Block(id=f"{source}:{start + 1}", text=plain, title=title, anchor=anchor, + source=source, line=start + 1, meta=block_meta)) + return blocks + + +# ── Line classification ────────────────────────────────────────────────── + +def _scan(lines): + """Yield (index, line, in_fence, heading_match) for every line.""" + in_fence = False + for i, line in enumerate(lines): + if _FENCE.match(line): + yield i, line, True, None + in_fence = not in_fence + continue + heading = None if in_fence else _HEADING.match(line) + yield i, line, in_fence, heading + + +def _split_file(lines): + title = "" + for _, _, in_fence, heading in _scan(lines): + if heading: + title = heading.group(2) + break + return [(0, len(lines), "file", "", title, "")] + + +def _split_heading(lines): + out = [] + start, section, title, anchor = 0, "", "", "" + for i, _, _, heading in _scan(lines): + if heading: + if i > start: + out.append((start, i, "section", section, title, anchor)) + start, section = i, heading.group(2) + title, anchor = section, slugify(section) + out.append((start, len(lines), "section", section, title, anchor)) + return out + + +def _split_list(lines): + out = [] + section = "" + cur = None # [start, kind] + last_content = -1 # last non-blank line of the current block + + def close(end): + nonlocal cur + if cur is not None: + out.append((cur[0], end, cur[1], section, "", "")) + cur = None + + prev_blank = True + for i, line, in_fence, heading in _scan(lines): + if heading: + close(last_content + 1) + section = heading.group(2) + prev_blank = True + continue + if not line.strip() and not in_fence: + if cur is not None and cur[1] == "paragraph": + close(last_content + 1) + prev_blank = True + continue + if in_fence: + if cur is None: + cur = [i, "paragraph"] + elif _LIST_ITEM.match(line): + close(last_content + 1) + cur = [i, "item"] + elif _HR.match(line): + close(last_content + 1) + prev_blank = True + continue + elif cur is not None and cur[1] == "item" and (line[:1] in (" ", "\t") or not prev_blank): + pass # indented or lazy continuation of the item + elif cur is None or cur[1] == "item": + close(last_content + 1) + cur = [i, "paragraph"] + last_content = i + prev_blank = False + close(last_content + 1) + return out + + +def _split_paragraph(lines): + out = [] + section = "" + start = None + for i, line, in_fence, heading in _scan(lines): + if heading: + if start is not None: + out.append((start, i, "paragraph", section, "", "")) + start = None + section = heading.group(2) + continue + if not line.strip() and not in_fence: + if start is not None: + out.append((start, i, "paragraph", section, "", "")) + start = None + continue + if _HR.match(line) and not in_fence: + continue + if start is None: + start = i + if start is not None: + out.append((start, len(lines), "paragraph", section, "", "")) + return out + + +# ── Markdown → plain text ──────────────────────────────────────────────── + +_IMAGE = re.compile(r"!\[([^\]]*)\]\([^)]*\)") +_LINK = re.compile(r"\[([^\]]+)\]\((?:[^()\s]|\([^()\s]*\))+(?:\s+\"[^\"]*\")?\)") +_REF_LINK = re.compile(r"\[([^\]]+)\]\[[^\]]*\]") +_AUTOLINK = re.compile(r"<(https?://[^>]+)>") +_BOLD = re.compile(r"(\*\*|__)(?=\S)(.+?)(?<=\S)\1") +_ITALIC_STAR = re.compile(r"(?\s?|(?:[-*+]|\d{1,9}[.)])\s+)") + + +def markdown_to_text(md: str) -> str: + """Remove Markdown syntax, keep line structure (blank lines collapsed).""" + out_lines = [] + for line in md.splitlines(): + if _FENCE.match(line) or _HR.match(line): + continue + line = _LINE_MARKER.sub("", line).strip() + line = _IMAGE.sub(r"\1", line) + line = _LINK.sub(r"\1", line) + line = _REF_LINK.sub(r"\1", line) + line = _AUTOLINK.sub(r"\1", line) + line = _CODE.sub(r"\1", line) + line = _BOLD.sub(r"\2", line) + line = _ITALIC_STAR.sub(r"\1", line) + line = _ITALIC_UNDER.sub(r"\1", line) + line = _STRIKE.sub(r"\1", line) + if line or (out_lines and out_lines[-1]): + out_lines.append(line) + return "\n".join(out_lines).strip() + + +def slugify(text: str) -> str: + """GitHub-style heading anchor.""" + text = markdown_to_text(text).lower() + text = re.sub(r"[^\w\- ]", "", text) + return re.sub(r" ", "-", text.strip()) + diff --git a/src/mdgrep/query.py b/src/mdgrep/query.py new file mode 100644 index 0000000..9749581 --- /dev/null +++ b/src/mdgrep/query.py @@ -0,0 +1,328 @@ +"""Query language: substrings, phrases, regular expressions, AND/OR/NOT and parentheses. + +Syntax (all matching is case-insensitive): + + word substring match ("rust" also matches "Rustacean") + "a phrase" exact phrase, any whitespace between the words + /regex/ regular expression (write \\/ for a literal slash) + a b, a AND b both (also: UND, &, &&) + a OR b either (also: ODER, |, ||) + NOT a, -a, !a negation (also: NICHT) + ( ... ) grouping + +Precedence: NOT binds tighter than AND, AND tighter than OR. +Operator keywords are only recognized in upper case; "and" is a normal search word. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field + +import regex + +__all__ = [ + "Query", + "QuerySyntaxError", + "QueryTimeout", + "compile_query", +] + +_FLAGS = regex.IGNORECASE | regex.UNICODE + +_KEYWORDS = { + "AND": "AND", "UND": "AND", + "OR": "OR", "ODER": "OR", + "NOT": "NOT", "NICHT": "NOT", +} + + +class QuerySyntaxError(ValueError): + """Invalid query. `code` is a stable identifier, `position` a 0-based offset into the query.""" + + def __init__(self, message: str, position: int, code: str): + super().__init__(f"{message} (at position {position})") + self.message = message + self.position = position + self.code = code + + def to_dict(self) -> dict: + return {"code": self.code, "message": self.message, "position": self.position} + + +class QueryTimeout(TimeoutError): + """Matching exceeded the time limit (e.g. a catastrophic regular expression).""" + + +# ── Tokens ──────────────────────────────────────────────────────────────── + +@dataclass +class _Token: + kind: str # LPAREN RPAREN AND OR NOT TERM + pos: int + value: str = "" + term_kind: str = "" # word | phrase | regex + + +def _tokenize(text: str) -> list[_Token]: + tokens: list[_Token] = [] + i, n = 0, len(text) + while i < n: + c = text[i] + if c.isspace(): + i += 1 + elif c in "()": + tokens.append(_Token("LPAREN" if c == "(" else "RPAREN", i)) + i += 1 + elif c in "&|": + kind = "AND" if c == "&" else "OR" + tokens.append(_Token(kind, i)) + i += 2 if text[i:i + 2] == c * 2 else 1 + elif c == "!" or (c == "-" and i + 1 < n and not text[i + 1].isspace()): + tokens.append(_Token("NOT", i)) + i += 1 + elif c == '"': + j = i + 1 + buf = [] + while j < n and text[j] != '"': + if text[j] == "\\" and j + 1 < n: + j += 1 + buf.append(text[j]) + j += 1 + if j >= n: + raise QuerySyntaxError("Unterminated phrase", i, "unterminated_phrase") + tokens.append(_Token("TERM", i, "".join(buf), "phrase")) + i = j + 1 + elif c == "/": + j = i + 1 + buf = [] + while j < n and text[j] != "/": + if text[j] == "\\" and j + 1 < n: + if text[j + 1] == "/": + buf.append("/") + else: + buf.append(text[j:j + 2]) + j += 2 + continue + buf.append(text[j]) + j += 1 + if j >= n: + raise QuerySyntaxError("Unterminated regular expression", i, "unterminated_regex") + tokens.append(_Token("TERM", i, "".join(buf), "regex")) + i = j + 1 + else: + j = i + while j < n and not text[j].isspace() and text[j] not in '()"': + j += 1 + word = text[i:j] + if word in _KEYWORDS: + tokens.append(_Token(_KEYWORDS[word], i)) + else: + tokens.append(_Token("TERM", i, word, "word")) + i = j + return tokens + + +# ── AST ─────────────────────────────────────────────────────────────────── + +@dataclass +class Term: + kind: str # word | phrase | regex + source: str + pos: int + pattern: regex.Pattern = field(repr=False) + + def evaluate(self, text: str, deadline: float | None) -> bool: + return self.pattern.search(text, timeout=_remaining(deadline)) is not None + + +@dataclass +class Not: + child: object + + def evaluate(self, text, deadline): + return not self.child.evaluate(text, deadline) + + +@dataclass +class And: + children: list + + def evaluate(self, text, deadline): + return all(c.evaluate(text, deadline) for c in self.children) + + +@dataclass +class Or: + children: list + + def evaluate(self, text, deadline): + return any(c.evaluate(text, deadline) for c in self.children) + + +def _remaining(deadline: float | None) -> float | None: + if deadline is None: + return None + left = deadline - time.monotonic() + if left <= 0: + raise QueryTimeout("Search time limit exceeded") + return left + + +def _compile_term(token: _Token) -> Term: + if token.term_kind == "regex": + if not token.value: + raise QuerySyntaxError("Empty regular expression", token.pos, "invalid_regex") + source = token.value + elif token.term_kind == "phrase": + words = token.value.split() + if not words: + raise QuerySyntaxError("Empty phrase", token.pos, "empty_phrase") + source = r"\s+".join(regex.escape(w) for w in words) + else: + source = regex.escape(token.value) + try: + pattern = regex.compile(source, _FLAGS) + except regex.error as e: + raise QuerySyntaxError(f"Invalid regular expression: {e.msg}", token.pos, "invalid_regex") from None + return Term(token.term_kind, token.value, token.pos, pattern) + + +# ── Parser (recursive descent) ──────────────────────────────────────────── + +class _Parser: + def __init__(self, tokens: list[_Token], length: int): + self.tokens = tokens + self.i = 0 + self.length = length + self.terms: list[tuple[Term, bool]] = [] # (term, negated) + + def peek(self) -> _Token | None: + return self.tokens[self.i] if self.i < len(self.tokens) else None + + def take(self) -> _Token: + tok = self.tokens[self.i] + self.i += 1 + return tok + + def end_pos(self) -> int: + return self.length + + def parse(self): + if not self.tokens: + raise QuerySyntaxError("Empty query", 0, "empty_query") + node = self.parse_or(negated=False) + tok = self.peek() + if tok is not None: + if tok.kind == "RPAREN": + raise QuerySyntaxError("Unexpected closing parenthesis", tok.pos, "unexpected_paren") + raise QuerySyntaxError("Unexpected operator", tok.pos, "unexpected_token") + return node + + def parse_or(self, negated): + children = [self.parse_and(negated)] + while (tok := self.peek()) is not None and tok.kind == "OR": + self.take() + children.append(self.parse_and(negated)) + return children[0] if len(children) == 1 else Or(children) + + def parse_and(self, negated): + children = [self.parse_not(negated)] + while (tok := self.peek()) is not None and tok.kind in ("AND", "NOT", "TERM", "LPAREN"): + if tok.kind == "AND": + self.take() + children.append(self.parse_not(negated)) + return children[0] if len(children) == 1 else And(children) + + def parse_not(self, negated): + tok = self.peek() + if tok is not None and tok.kind == "NOT": + self.take() + return Not(self.parse_not(not negated)) + return self.parse_primary(negated) + + def parse_primary(self, negated): + tok = self.peek() + if tok is None: + raise QuerySyntaxError("Search term expected", self.end_pos(), "missing_operand") + if tok.kind == "TERM": + self.take() + term = _compile_term(tok) + self.terms.append((term, negated)) + return term + if tok.kind == "LPAREN": + self.take() + if (nxt := self.peek()) is not None and nxt.kind == "RPAREN": + raise QuerySyntaxError("Empty parentheses", tok.pos, "empty_group") + node = self.parse_or(negated) + close = self.peek() + if close is None or close.kind != "RPAREN": + raise QuerySyntaxError("Missing closing parenthesis", tok.pos, "unclosed_paren") + self.take() + return node + if tok.kind == "RPAREN": + raise QuerySyntaxError("Unexpected closing parenthesis", tok.pos, "unexpected_paren") + raise QuerySyntaxError("Search term expected", tok.pos, "missing_operand") + + +# ── Public API ──────────────────────────────────────────────────────────── + +class Query: + """A compiled query. Use `matches()` to test a text and `spans()` for highlighting.""" + + def __init__(self, text: str, root, terms: list[tuple[Term, bool]]): + self.text = text + self._root = root + self._terms = terms + + @property + def positive_terms(self) -> list[Term]: + """Terms that contribute to a match (not under an odd number of NOTs).""" + return [t for t, negated in self._terms if not negated] + + def matches(self, text: str, timeout: float | None = None, *, deadline: float | None = None) -> bool: + """True if `text` satisfies the query. Raises QueryTimeout when the time limit is exceeded.""" + if deadline is None and timeout is not None: + deadline = time.monotonic() + timeout + try: + return self._root.evaluate(text, deadline) + except TimeoutError as e: + raise QueryTimeout(str(e) or "Search time limit exceeded") from None + + def spans(self, text: str, timeout: float | None = None, *, deadline: float | None = None) -> list[tuple[int, int]]: + """Sorted, merged (start, end) ranges of all positive term matches in `text`.""" + if deadline is None and timeout is not None: + deadline = time.monotonic() + timeout + found = [] + try: + for term in self.positive_terms: + for m in term.pattern.finditer(text, timeout=_remaining(deadline)): + if m.end() > m.start(): + found.append((m.start(), m.end())) + except TimeoutError as e: + raise QueryTimeout(str(e) or "Search time limit exceeded") from None + return _merge(found) + + def __repr__(self): + return f"Query({self.text!r})" + + +def _merge(spans: list[tuple[int, int]]) -> list[tuple[int, int]]: + merged: list[list[int]] = [] + for start, end in sorted(spans): + if merged and start <= merged[-1][1]: + merged[-1][1] = max(merged[-1][1], end) + else: + merged.append([start, end]) + return [(s, e) for s, e in merged] + + +def compile_query(text: str, *, max_length: int = 500, max_terms: int = 50) -> Query: + """Parse and compile a query string. Raises QuerySyntaxError on invalid input.""" + if len(text) > max_length: + raise QuerySyntaxError(f"Query longer than {max_length} characters", max_length, "too_long") + parser = _Parser(_tokenize(text), len(text)) + root = parser.parse() + if len(parser.terms) > max_terms: + raise QuerySyntaxError(f"More than {max_terms} search terms", 0, "too_many_terms") + return Query(text, root, parser.terms) diff --git a/src/mdgrep/snippets.py b/src/mdgrep/snippets.py new file mode 100644 index 0000000..5cd66ce --- /dev/null +++ b/src/mdgrep/snippets.py @@ -0,0 +1,93 @@ +"""Turn a text plus match spans into display segments, shortened around the hits if needed.""" + +from __future__ import annotations + +__all__ = ["segments", "ELLIPSIS"] + +ELLIPSIS = " … " + + +def segments(text: str, spans: list[tuple[int, int]], *, max_chars: int | None = None, + context: int = 120) -> list[dict]: + """Split `text` into [{"t": str, "hit": bool}, ...]. + + If `max_chars` is set and the text is longer, only windows of `context` characters around + the hits are kept (joined by ELLIPSIS segments). Without hits the beginning of the text is shown. + """ + if max_chars is None or len(text) <= max_chars: + return _split(text, spans, 0, len(text)) + + windows = _windows(text, spans, context, max_chars) + out: list[dict] = [] + for i, (start, end) in enumerate(windows): + if start > 0 or i > 0: + out.append({"t": ELLIPSIS, "hit": False}) + out.extend(_split(text, spans, start, end)) + if windows and windows[-1][1] < len(text): + out.append({"t": ELLIPSIS, "hit": False}) + return out + + +def _windows(text, spans, context, max_chars): + if not spans: + return [(0, _snap_end(text, max_chars))] + windows: list[list[int]] = [] + budget = max_chars + for start, end in spans: + w_start = _snap_start(text, max(0, start - context)) + w_end = _snap_end(text, min(len(text), end + context)) + if windows and w_start <= windows[-1][1]: + grow = w_end - windows[-1][1] + if grow > 0: + if grow > budget: + break + windows[-1][1] = w_end + budget -= grow + continue + size = w_end - w_start + if size > budget: + if not windows: # always show the first hit + windows.append([w_start, w_end]) + break + windows.append([w_start, w_end]) + budget -= size + return [(s, e) for s, e in windows] + + +def _snap_start(text, pos): + """Move a window start back to the beginning of a word.""" + if pos <= 0: + return 0 + limit = max(0, pos - 20) + i = pos + while i > limit and not text[i - 1].isspace(): + i -= 1 + return i if i > limit else pos + + +def _snap_end(text, pos): + """Move a window end forward to the end of a word.""" + n = len(text) + if pos >= n: + return n + limit = min(n, pos + 20) + i = pos + while i < limit and not text[i].isspace(): + i += 1 + return i + + +def _split(text, spans, start, end): + out = [] + pos = start + for s, e in spans: + if e <= start or s >= end: + continue + s, e = max(s, start), min(e, end) + if s > pos: + out.append({"t": text[pos:s], "hit": False}) + out.append({"t": text[s:e], "hit": True}) + pos = e + if pos < end: + out.append({"t": text[pos:end], "hit": False}) + return out diff --git a/src/mdgrep/static/mdgrep-widget.css b/src/mdgrep/static/mdgrep-widget.css new file mode 100644 index 0000000..df211aa --- /dev/null +++ b/src/mdgrep/static/mdgrep-widget.css @@ -0,0 +1,79 @@ +/* mdgrep search widget v0.1.0 – MIT License + * + * Theming: set these custom properties on :root or any ancestor + * --mdgrep-text --mdgrep-muted --mdgrep-surface --mdgrep-border + * --mdgrep-accent --mdgrep-mark-bg --mdgrep-mark-text --mdgrep-error --mdgrep-radius + */ + +.mdgrep { color: var(--mdgrep-text, #1a1d2e); } +.mdgrep [hidden] { display: none !important; } + +.mdgrep-form { display: flex; flex-wrap: wrap; gap: 0.5rem; } + +.mdgrep-input, +.mdgrep-filter { + font: inherit; + color: inherit; + background: var(--mdgrep-surface, #fff); + border: 1px solid var(--mdgrep-border, #d5d8e3); + border-radius: var(--mdgrep-radius, 0.5rem); + padding: 0.5rem 0.75rem; +} +.mdgrep-input { flex: 1 1 16rem; min-width: 0; } +.mdgrep-filter { flex: 0 0 auto; cursor: pointer; } +.mdgrep-input:focus, +.mdgrep-filter:focus { outline: 2px solid var(--mdgrep-accent, #4a7cff); outline-offset: 1px; } + +.mdgrep-help { margin: 0.75rem 0 0; font-size: 0.875em; color: var(--mdgrep-muted, #6b7394); } +.mdgrep-help summary { cursor: pointer; width: max-content; } +.mdgrep-help table { border-collapse: collapse; margin: 0.5rem 0; background: none; border: none; } +.mdgrep-help td { border: none; padding: 0.15rem 1rem 0.15rem 0; vertical-align: top; } +.mdgrep-help code { white-space: nowrap; } + +.mdgrep-status { margin: 0.75rem 0; min-height: 1.4em; font-size: 0.875em; color: var(--mdgrep-muted, #6b7394); } +.mdgrep-status-error { color: var(--mdgrep-error, #c0392b); } +.mdgrep-status-warn { color: var(--mdgrep-error, #c0392b); } +.mdgrep-busy .mdgrep-results { opacity: 0.6; } + +.mdgrep-results { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 0.75rem; } + +.mdgrep-card { + margin: 0; + padding: 0.75rem 1rem; + background: var(--mdgrep-surface, #fff); + border: 1px solid var(--mdgrep-border, #d5d8e3); + border-radius: var(--mdgrep-radius, 0.5rem); + overflow-wrap: anywhere; +} +.mdgrep-card:hover { border-color: var(--mdgrep-accent, #4a7cff); } + +.mdgrep-card-head { display: flex; flex-wrap: wrap; align-items: center; gap: 0.5rem; margin-bottom: 0.35rem; + font-size: 0.8em; color: var(--mdgrep-muted, #6b7394); } +.mdgrep-badge { padding: 0 0.45rem; border: 1px solid var(--mdgrep-border, #d5d8e3); + border-radius: 999px; text-transform: uppercase; letter-spacing: 0.03em; } +.mdgrep-date { font-style: italic; } + +.mdgrep-title { display: block; font-weight: 700; margin-bottom: 0.3rem; text-decoration: none; } +.mdgrep-snippet { display: block; white-space: pre-line; color: inherit; text-decoration: none; font-size: 0.925em; } +a.mdgrep-snippet:visited { color: inherit; } +.mdgrep-card a:hover { text-decoration: none; } + +.mdgrep mark { background: var(--mdgrep-mark-bg, #ffe58a); color: var(--mdgrep-mark-text, inherit); + border-radius: 2px; padding: 0 1px; } + +.mdgrep-also { margin-top: 0.5rem; font-size: 0.8em; color: var(--mdgrep-muted, #6b7394); } +.mdgrep-also summary { cursor: pointer; width: max-content; } +.mdgrep-also ul { margin: 0.35rem 0 0; padding-left: 1.25rem; } + +.mdgrep-more { + display: block; + margin: 1rem auto 0; + font: inherit; + color: inherit; + background: var(--mdgrep-surface, #fff); + border: 1px solid var(--mdgrep-border, #d5d8e3); + border-radius: var(--mdgrep-radius, 0.5rem); + padding: 0.4rem 1.25rem; + cursor: pointer; +} +.mdgrep-more:hover { border-color: var(--mdgrep-accent, #4a7cff); } diff --git a/src/mdgrep/static/mdgrep-widget.js b/src/mdgrep/static/mdgrep-widget.js new file mode 100644 index 0000000..56756e1 --- /dev/null +++ b/src/mdgrep/static/mdgrep-widget.js @@ -0,0 +1,399 @@ +/*! mdgrep search widget v0.1.0 – MIT License + * + * Usage: + * + * + * + * + * + * Or without script:
+ * (filters as JSON in data-filters). + * + * Expects the JSON contract of mdgrep's Web API (SearchResult.to_dict()). + * All result text is inserted with textContent – no HTML from the server is interpreted. + */ +(function (global) { + 'use strict'; + + var LABELS = { + en: { + placeholder: 'Search… e.g. (rust OR go) AND NOT crypto', + inputLabel: 'Search query', + help: 'Search syntax', + helpRows: [ + ['word', 'part of a word, upper/lower case ignored'], + ['"exact phrase"', 'words in this order'], + ['/regex/', 'regular expression, e.g. /llms?\\b/'], + ['a b a AND b', 'both terms (also &)'], + ['a OR b', 'at least one term (also |)'], + ['NOT a -a !a', 'term must not occur'], + ['( … )', 'grouping, e.g. (a OR b) AND c'], + ], + minChars: function (n) { return 'Enter at least ' + n + ' characters.'; }, + searching: 'Searching…', + none: 'No matches.', + count: function (groups, blocks, ms) { + return groups + (groups === 1 ? ' result' : ' results') + + (blocks !== groups ? ' (' + blocks + ' matches)' : '') + ' · ' + Math.round(ms) + ' ms'; + }, + truncated: 'Time limit reached – results are incomplete.', + more: 'Load more', + alsoIn: function (n) { return 'also in ' + n + (n === 1 ? ' other place' : ' other places'); }, + errorAt: function (msg, pos) { return msg + ' (position ' + (pos + 1) + ')'; }, + rateLimited: 'Too many requests – please wait a moment.', + failed: 'Search failed. Please try again later.', + errors: { + empty_query: 'Empty query', + empty_phrase: 'Empty phrase', + unterminated_phrase: 'Closing quote missing', + unterminated_regex: 'Closing slash of the regular expression missing', + invalid_regex: 'Invalid regular expression', + missing_operand: 'Search term expected', + unexpected_paren: 'Unexpected closing parenthesis', + unexpected_token: 'Unexpected operator', + unclosed_paren: 'Closing parenthesis missing', + empty_group: 'Empty parentheses', + too_long: 'Query too long', + too_many_terms: 'Too many search terms', + }, + }, + de: { + placeholder: 'Suchen… z. B. (rust ODER go) UND NICHT crypto', + inputLabel: 'Suchbegriff', + help: 'Suchsyntax', + helpRows: [ + ['wort', 'Wortbestandteil, Groß-/Kleinschreibung egal'], + ['"genaue phrase"', 'Wörter in dieser Reihenfolge'], + ['/regex/', 'regulärer Ausdruck, z. B. /llms?\\b/'], + ['a b a UND b', 'beide Begriffe (auch AND, &)'], + ['a ODER b', 'mindestens ein Begriff (auch OR, |)'], + ['NICHT a -a !a', 'Begriff darf nicht vorkommen (auch NOT)'], + ['( … )', 'Klammern, z. B. (a ODER b) UND c'], + ], + minChars: function (n) { return 'Bitte mindestens ' + n + ' Zeichen eingeben.'; }, + searching: 'Suche läuft…', + none: 'Keine Treffer.', + count: function (groups, blocks, ms) { + return groups + (groups === 1 ? ' Ergebnis' : ' Ergebnisse') + + (blocks !== groups ? ' (' + blocks + ' Fundstellen)' : '') + ' · ' + Math.round(ms) + ' ms'; + }, + truncated: 'Zeitlimit erreicht – die Ergebnisse sind unvollständig.', + more: 'Mehr laden', + alsoIn: function (n) { return 'auch in ' + n + (n === 1 ? ' weiterem Bericht' : ' weiteren Berichten'); }, + errorAt: function (msg, pos) { return msg + ' (Position ' + (pos + 1) + ')'; }, + rateLimited: 'Zu viele Anfragen – bitte einen Moment warten.', + failed: 'Die Suche ist fehlgeschlagen. Bitte später erneut versuchen.', + errors: { + empty_query: 'Leere Suchanfrage', + empty_phrase: 'Leere Phrase', + unterminated_phrase: 'Schließendes Anführungszeichen fehlt', + unterminated_regex: 'Schließender Schrägstrich des regulären Ausdrucks fehlt', + invalid_regex: 'Ungültiger regulärer Ausdruck', + missing_operand: 'Suchbegriff erwartet', + unexpected_paren: 'Unerwartete schließende Klammer', + unexpected_token: 'Unerwarteter Operator', + unclosed_paren: 'Schließende Klammer fehlt', + empty_group: 'Leere Klammern', + too_long: 'Suchanfrage zu lang', + too_many_terms: 'Zu viele Suchbegriffe', + }, + }, + }; + + var DEFAULTS = { + endpoint: 'search', + filters: [], // [{name, label, options: [{value, label}], value}] + labels: 'en', // 'en' | 'de' | object overriding single labels + live: true, + debounce: 300, + minChars: 2, + limit: 30, + syncUrl: true, // keep q and filters in the page URL (shareable, back button) + autofocus: true, + help: true, + hrefField: 'href', // block.meta[hrefField] is the link target + href: null, // function(block) → URL, overrides hrefField + dateField: null, // block.meta[dateField] shown in the card header + badgeFields: [], // block.meta keys shown as badges + formatBadge: null, // function(field, value, block) → text + alsoText: null, // function(block) → link text in the "also in" list + }; + + function el(tag, className, text) { + var node = document.createElement(tag); + if (className) node.className = className; + if (text != null) node.textContent = text; + return node; + } + + function appendSegments(parent, segments) { + (segments || []).forEach(function (s) { + parent.appendChild(s.hit ? el('mark', null, s.t) : document.createTextNode(s.t)); + }); + } + + function resolveLabels(labels) { + if (typeof labels === 'string') return LABELS[labels] || LABELS.en; + var base = LABELS[(labels && labels.lang) || 'en'] || LABELS.en; + var out = {}; + Object.keys(base).forEach(function (k) { out[k] = base[k]; }); + Object.keys(labels || {}).forEach(function (k) { out[k] = labels[k]; }); + return out; + } + + function mount(root, options) { + var o = {}; + Object.keys(DEFAULTS).forEach(function (k) { o[k] = DEFAULTS[k]; }); + Object.keys(options || {}).forEach(function (k) { o[k] = options[k]; }); + var L = resolveLabels(o.labels); + + // ── DOM ────────────────────────────────────────────────────────────── + root.classList.add('mdgrep'); + root.textContent = ''; + var form = el('form', 'mdgrep-form'); + form.setAttribute('role', 'search'); + var input = el('input', 'mdgrep-input'); + input.type = 'search'; + input.name = 'q'; + input.placeholder = L.placeholder; + input.setAttribute('aria-label', L.inputLabel); + input.autocomplete = 'off'; + input.spellcheck = false; + form.appendChild(input); + + var selects = {}; + o.filters.forEach(function (f) { + var sel = el('select', 'mdgrep-filter'); + sel.name = f.name; + sel.setAttribute('aria-label', f.label || f.name); + (f.options || []).forEach(function (opt) { + var option = el('option', null, opt.label); + option.value = opt.value; + sel.appendChild(option); + }); + if (f.value != null) sel.value = f.value; + selects[f.name] = sel; + form.appendChild(sel); + }); + root.appendChild(form); + + if (o.help) { + var help = el('details', 'mdgrep-help'); + help.appendChild(el('summary', null, L.help)); + var table = el('table'); + L.helpRows.forEach(function (row) { + var tr = el('tr'); + var code = el('td'); + code.appendChild(el('code', null, row[0])); + tr.appendChild(code); + tr.appendChild(el('td', null, row[1])); + table.appendChild(tr); + }); + help.appendChild(table); + root.appendChild(help); + } + + var status = el('div', 'mdgrep-status'); + status.setAttribute('aria-live', 'polite'); + root.appendChild(status); + var list = el('ol', 'mdgrep-results'); + root.appendChild(list); + var more = el('button', 'mdgrep-more', L.more); + more.type = 'button'; + more.hidden = true; + root.appendChild(more); + + // ── State from URL ─────────────────────────────────────────────────── + if (o.syncUrl) { + var params = new URLSearchParams(global.location.search); + if (params.has('q')) input.value = params.get('q'); + Object.keys(selects).forEach(function (name) { + if (params.has(name)) selects[name].value = params.get(name); + }); + } + + var controller = null; + var timer = null; + var nextOffset = null; + + function filterValues() { + var out = {}; + Object.keys(selects).forEach(function (name) { out[name] = selects[name].value; }); + return out; + } + + function updateUrl(q) { + if (!o.syncUrl) return; + var url = new URL(global.location.href); + if (q) url.searchParams.set('q', q); else url.searchParams.delete('q'); + var values = filterValues(); + Object.keys(values).forEach(function (name) { + if (values[name]) url.searchParams.set(name, values[name]); else url.searchParams.delete(name); + }); + global.history.replaceState(null, '', url); + } + + function setStatus(text, kind) { + status.textContent = text || ''; + status.className = 'mdgrep-status' + (kind ? ' mdgrep-status-' + kind : ''); + } + + function hrefOf(block) { + if (o.href) return o.href(block); + var meta = block.meta || {}; + if (meta[o.hrefField]) return meta[o.hrefField]; + return block.anchor ? '#' + block.anchor : null; + } + + function link(block, className) { + var href = hrefOf(block); + var a = el(href ? 'a' : 'div', className); + if (href) a.href = href; + return a; + } + + function renderGroup(r) { + var b = r.block; + var meta = b.meta || {}; + var li = el('li', 'mdgrep-card'); + + var head = el('div', 'mdgrep-card-head'); + o.badgeFields.forEach(function (field) { + if (meta[field] == null || meta[field] === '') return; + var text = o.formatBadge ? o.formatBadge(field, meta[field], b) : String(meta[field]); + if (text) head.appendChild(el('span', 'mdgrep-badge mdgrep-badge-' + field, text)); + }); + if (o.dateField && meta[o.dateField]) head.appendChild(el('span', 'mdgrep-date', meta[o.dateField])); + if (head.childNodes.length) li.appendChild(head); + + if (b.title) { + var title = link(b, 'mdgrep-title'); + appendSegments(title, b.title_segments); + li.appendChild(title); + } + var snippet = link(b, 'mdgrep-snippet'); + appendSegments(snippet, b.segments); + li.appendChild(snippet); + + if (r.also && r.also.length) { + var details = el('details', 'mdgrep-also'); + details.appendChild(el('summary', null, L.alsoIn(r.also.length))); + var ul = el('ul'); + r.also.forEach(function (other) { + var item = el('li'); + var a = link(other, null); + var m = other.meta || {}; + a.textContent = o.alsoText ? o.alsoText(other) : (o.dateField && m[o.dateField]) || other.title || other.id; + item.appendChild(a); + ul.appendChild(item); + }); + details.appendChild(ul); + li.appendChild(details); + } + return li; + } + + function search(append) { + var q = input.value.trim(); + if (controller) controller.abort(); + clearTimeout(timer); + updateUrl(q); + if (!append) { list.textContent = ''; more.hidden = true; nextOffset = null; } + if (q.length < o.minChars) { + setStatus(q ? L.minChars(o.minChars) : ''); + return Promise.resolve(null); + } + + var url = new URL(o.endpoint, global.location.href); + url.searchParams.set('q', q); + var values = filterValues(); + Object.keys(values).forEach(function (name) { if (values[name]) url.searchParams.set(name, values[name]); }); + url.searchParams.set('limit', o.limit); + if (append && nextOffset) url.searchParams.set('offset', nextOffset); + + controller = new AbortController(); + var current = controller; + if (!append) setStatus(L.searching, 'busy'); + root.classList.add('mdgrep-busy'); + + return fetch(url, { signal: current.signal, headers: { Accept: 'application/json' } }) + .then(function (resp) { + if (resp.status === 429) throw { userMessage: L.rateLimited }; + return resp.json().then(function (data) { + if (!resp.ok) { + var err = data && data.error; + if (!err) throw { userMessage: L.failed }; + throw { userMessage: L.errorAt(L.errors[err.code] || err.message, err.position), position: err.position }; + } + return data; + }, function () { throw { userMessage: L.failed }; }); + }) + .then(function (data) { + if (current !== controller) return data; + data.results.forEach(function (r) { list.appendChild(renderGroup(r)); }); + nextOffset = data.next_offset; + more.hidden = nextOffset == null; + if (!data.total_groups) setStatus(L.none); + else setStatus(L.count(data.total_groups, data.total_blocks, data.elapsed_ms) + + (data.truncated ? ' · ' + L.truncated : ''), data.truncated ? 'warn' : null); + root.dispatchEvent(new CustomEvent('mdgrep:results', { detail: data })); + return data; + }) + .catch(function (e) { + if (e && e.name === 'AbortError') return null; + setStatus((e && e.userMessage) || L.failed, 'error'); + return null; + }) + .then(function (data) { + if (current === controller) root.classList.remove('mdgrep-busy'); + return data; + }); + } + + input.addEventListener('input', function () { + if (!o.live) return; + clearTimeout(timer); + timer = setTimeout(function () { search(false); }, o.debounce); + }); + form.addEventListener('submit', function (e) { e.preventDefault(); search(false); }); + Object.keys(selects).forEach(function (name) { + selects[name].addEventListener('change', function () { search(false); }); + }); + more.addEventListener('click', function () { search(true); }); + + if (o.autofocus) input.focus(); + if (input.value.trim()) search(false); + + return { + search: function () { return search(false); }, + setQuery: function (q) { input.value = q; return search(false); }, + input: input, + }; + } + + function autoMount() { + var nodes = document.querySelectorAll('[data-mdgrep]:not(.mdgrep)'); + Array.prototype.forEach.call(nodes, function (node) { + var d = node.dataset; + mount(node, { + endpoint: d.endpoint || DEFAULTS.endpoint, + labels: d.labels || DEFAULTS.labels, + filters: d.filters ? JSON.parse(d.filters) : [], + dateField: d.dateField || null, + badgeFields: d.badgeFields ? d.badgeFields.split(',') : [], + }); + }); + } + + global.MdGrep = { mount: mount, labels: LABELS, version: '0.1.0' }; + + if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', autoMount); + else autoMount(); +})(window); diff --git a/src/mdgrep/web.py b/src/mdgrep/web.py new file mode 100644 index 0000000..e828ca0 --- /dev/null +++ b/src/mdgrep/web.py @@ -0,0 +1,161 @@ +"""Flask integration: a JSON search endpoint plus the widget's static files. + + from mdgrep import SearchEngine, load_markdown + from mdgrep.web import create_app + + engine = SearchEngine(lambda: load_markdown("docs", split="list")) + app = create_app(engine, filters=["lang"]) # GET /search?q=...&lang=... + +Or mount into an existing app: + + app.register_blueprint(create_blueprint(engine), url_prefix="/api") + +Requires the optional dependency: pip install "mdgrep[web]". +""" + +from __future__ import annotations + +import argparse +import time +from pathlib import Path +from typing import Any, Callable, Hashable, Iterable + +from flask import Blueprint, Flask, jsonify, request, send_from_directory + +from .engine import Block, SearchEngine +from .markdown import SPLIT_MODES, load_markdown +from .query import QuerySyntaxError + +__all__ = ["create_blueprint", "create_app", "STATIC_DIR"] + +STATIC_DIR = Path(__file__).parent / "static" + + +def create_blueprint( + engine: SearchEngine, + *, + filters: Iterable[str] = (), + group_by: Callable[[Block], Hashable | None] | None = None, + sort_key: Callable[[Block], Any] | None = None, + reverse: bool = False, + default_limit: int = 30, + max_limit: int = 100, + time_budget: float = 1.5, + snippet_chars: int | None = 700, + context: int = 120, + max_query_length: int = 300, + hide_meta: Iterable[str] = ("markdown",), + name: str = "mdgrep", +) -> Blueprint: + """Blueprint with `GET /search` and the widget assets under `/mdgrep-static/`. + + Query parameters: q, offset, limit and one parameter per name in `filters` + (empty or "all" = no restriction, comma-separated = several values). + Errors: HTTP 400 with {"error": {"code", "message", "position"}}. + """ + bp = Blueprint(name, __name__, static_folder=str(STATIC_DIR), static_url_path="/mdgrep-static") + filter_names = list(filters) + hidden = set(hide_meta) + + @bp.get("/search") + def search(): + q = request.args.get("q", "").strip() + offset = _int_arg("offset", 0, 0, 10**9) + limit = _int_arg("limit", default_limit, 1, max_limit) + if not q: + return jsonify(_empty(offset, limit)) + if len(q) > max_query_length: + return _error("too_long", f"Query longer than {max_query_length} characters", max_query_length) + + selected = {} + for fname in filter_names: + value = request.args.get(fname, "").strip() + if value and value != "all": + selected[fname] = value.split(",") if "," in value else value + + try: + result = engine.search(q, filters=selected, group_by=group_by, sort_key=sort_key, + reverse=reverse, offset=offset, limit=limit, + time_budget=time_budget, snippet_chars=snippet_chars, + context=context) + except QuerySyntaxError as e: + return _error(e.code, e.message, e.position) + + data = result.to_dict() + if hidden: + for r in data["results"]: + for b in [r["block"], *r["also"]]: + for key in hidden: + b["meta"].pop(key, None) + return jsonify(data) + + return bp + + +def create_app(engine: SearchEngine, *, url_prefix: str = "", static_dir: str | Path | None = None, + **blueprint_options) -> Flask: + """Standalone Flask app. With `static_dir`, files from that directory are served at `/` + (handy for local testing of a static site together with its search API).""" + app = Flask(__name__, static_folder=None) + app.json.ensure_ascii = False + app.register_blueprint(create_blueprint(engine, **blueprint_options), url_prefix=url_prefix or None) + + if static_dir is not None: + root = Path(static_dir).resolve() + + @app.get("/", defaults={"path": "index.html"}) + @app.get("/") + def static_files(path): + if (root / path).is_dir(): + path = f"{path.rstrip('/')}/index.html" + return send_from_directory(root, path) + + return app + + +def _int_arg(name, default, lo, hi): + try: + value = int(request.args.get(name, default)) + except (TypeError, ValueError): + value = default + return max(lo, min(hi, value)) + + +def _empty(offset, limit): + return {"query": "", "total_groups": 0, "total_blocks": 0, "offset": offset, "limit": limit, + "next_offset": None, "truncated": False, "elapsed_ms": 0, "results": []} + + +def _error(code, message, position): + return jsonify({"error": {"code": code, "message": message, "position": position}}), 400 + + +def main(argv=None): + """Development server: python -m mdgrep.web --dir docs [--static site] [--port 8000]""" + p = argparse.ArgumentParser(prog="python -m mdgrep.web", description="mdgrep development server") + p.add_argument("--dir", required=True, help="directory with Markdown files") + p.add_argument("--split", choices=SPLIT_MODES, default="paragraph") + p.add_argument("--static", help="serve this directory at / (API then lives under --api-prefix)") + p.add_argument("--api-prefix", default="/api") + p.add_argument("--filters", default="", help="comma-separated meta keys usable as filters") + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, default=8000) + args = p.parse_args(argv) + + md_dir = Path(args.dir) + engine = SearchEngine(lambda: load_markdown(md_dir, split=args.split, relative_to=md_dir), + reload_check=lambda: _fingerprint(md_dir), reload_interval=5) + filters = [f for f in args.filters.split(",") if f] + app = create_app(engine, url_prefix=args.api_prefix if args.static else "", + static_dir=args.static, filters=filters) + print(f"mdgrep: {len(engine.blocks)} blocks from {md_dir}") + app.run(host=args.host, port=args.port, debug=False) + + +def _fingerprint(directory: Path): + stats = [p.stat() for p in directory.rglob("*.md")] + return len(stats), max((s.st_mtime for s in stats), default=0) + + +if __name__ == "__main__": + main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..c1fc6d3 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,60 @@ +import json + +import pytest + +from mdgrep.cli import main + + +@pytest.fixture +def docs(tmp_path): + (tmp_path / "a.md").write_text("# A\n\nRust is fast.\n\nGo is simple.\n", encoding="utf-8") + (tmp_path / "b.md").write_text("- rust crypto wallet\n- python typing\n", encoding="utf-8") + return tmp_path + + +def run(capsys, *args): + code = main([*args, "--color", "never"]) + out, err = capsys.readouterr() + return code, out, err + + +def test_match_output_and_exit_code(capsys, docs): + code, out, _ = run(capsys, "rust -crypto", str(docs)) + assert code == 0 + assert f"{docs / 'a.md'}:3" in out + assert "Rust is fast." in out + assert "crypto" not in out + + +def test_no_match_exit_1(capsys, docs): + code, out, _ = run(capsys, "haskell", str(docs)) + assert code == 1 and out == "" + + +def test_syntax_error_exit_2_with_caret(capsys, docs): + code, _, err = run(capsys, "(rust", str(docs)) + assert code == 2 + assert "Missing closing parenthesis" in err and "^" in err + + +def test_count_and_files(capsys, docs): + assert run(capsys, "rust OR python", str(docs))[0] == 0 + _, out, _ = run(capsys, "rust OR python", str(docs), "--split", "list", "-c") + assert out.strip() == "3" + _, out, _ = run(capsys, "rust", str(docs), "-l") + assert out.split() == [str(docs / "a.md"), str(docs / "b.md")] + + +def test_json_output(capsys, docs): + code, out, _ = run(capsys, "/fa.t/", str(docs), "--json") + data = json.loads(out) + assert code == 0 and data["total_blocks"] == 1 + block = data["results"][0]["block"] + assert {"t": "fast", "hit": True} in block["segments"] + assert "markdown" not in block["meta"] + + +def test_color_highlight(capsys, docs): + main(["rust", str(docs / "a.md"), "--color", "always"]) + out = capsys.readouterr().out + assert "\033[1;31mRust\033[0m" in out diff --git a/tests/test_engine.py b/tests/test_engine.py new file mode 100644 index 0000000..5d4b415 --- /dev/null +++ b/tests/test_engine.py @@ -0,0 +1,116 @@ +import pytest + +from mdgrep import Block, QuerySyntaxError, SearchEngine + + +def blk(i, text, **meta): + return Block(id=str(i), text=text, title=f"T{i} {text.split()[0]}", meta=meta) + + +BLOCKS = [ + blk(1, "rust compiler news", lang="en", date="2026-01-01", url="u1"), + blk(2, "rust compiler news again", lang="en", date="2026-02-01", url="u1"), + blk(3, "Rust Übersetzer", lang="de", date="2026-01-15", url="u1"), + blk(4, "go generics", lang="en", date="2026-03-01", url="u2"), + blk(5, "python typing and rust bindings", lang="en", date="2025-12-01", url="u3"), +] + + +def ids(result): + return [g.block.id for g in result.groups] + + +def test_basic_search_original_order(): + res = SearchEngine(BLOCKS).search("rust") + assert ids(res) == ["1", "2", "3", "5"] + assert res.total_blocks == 4 and res.total_groups == 4 + + +def test_filters_single_and_multiple_values(): + eng = SearchEngine(BLOCKS) + assert ids(eng.search("rust", filters={"lang": "de"})) == ["3"] + assert ids(eng.search("rust", filters={"lang": ["de", "en"]})) == ["1", "2", "3", "5"] + assert ids(eng.search("rust", filters={"lang": ""})) == ["1", "2", "3", "5"] + + +def test_predicate(): + res = SearchEngine(BLOCKS).search("rust", predicate=lambda b: b.id != "1") + assert ids(res) == ["2", "3", "5"] + + +def test_sort_newest_first(): + res = SearchEngine(BLOCKS).search("rust", sort_key=lambda b: b.meta["date"], reverse=True) + assert ids(res) == ["2", "3", "1", "5"] + + +def test_grouping_uses_first_block_after_sorting(): + res = SearchEngine(BLOCKS).search( + "rust", sort_key=lambda b: b.meta["date"], reverse=True, + group_by=lambda b: (b.meta["lang"], b.meta["url"])) + assert ids(res) == ["2", "3", "5"] + assert [b.id for b in res.groups[0].also] == ["1"] + assert res.total_groups == 3 and res.total_blocks == 4 + + +def test_title_is_searched(): + eng = SearchEngine([Block("t", "body text", title="Special Title")]) + res = eng.search("special") + assert ids(res) == ["t"] + assert res.groups[0].title_segments == [{"t": "Special", "hit": True}, {"t": " Title", "hit": False}] + assert res.groups[0].hits == 0 # hits counts snippet matches + + +def test_paging(): + eng = SearchEngine(BLOCKS) + first = eng.search("rust", limit=2) + assert ids(first) == ["1", "2"] and first.next_offset == 2 + second = eng.search("rust", limit=2, offset=first.next_offset) + assert ids(second) == ["3", "5"] and second.next_offset is None + + +def test_segments_and_title_segments(): + res = SearchEngine(BLOCKS).search("compiler") + d = res.to_dict()["results"][0]["block"] + assert {"t": "compiler", "hit": True} in d["segments"] + assert d["hits"] == 1 + assert d["title_segments"] == [{"t": "T1 rust", "hit": False}] + + +def test_to_dict_contract(): + d = SearchEngine(BLOCKS).search("go", limit=10).to_dict() + assert set(d) == {"query", "total_groups", "total_blocks", "offset", "limit", + "next_offset", "truncated", "elapsed_ms", "results"} + r = d["results"][0] + assert set(r) == {"block", "also"} + assert r["block"]["meta"] == {"lang": "en", "date": "2026-03-01", "url": "u2"} + + +def test_syntax_error_propagates(): + with pytest.raises(QuerySyntaxError): + SearchEngine(BLOCKS).search("(rust") + + +def test_time_budget_truncates(): + blocks = [Block(id=str(i), text="a" * 30 + "!") for i in range(50)] + res = SearchEngine(blocks).search("/(a|aa)+$/", time_budget=0.3) + assert res.truncated + assert res.elapsed_ms < 3000 + + +def test_reload_on_token_change(): + state = {"token": 1, "blocks": [Block("a", "first")]} + eng = SearchEngine(lambda: list(state["blocks"]), reload_check=lambda: state["token"], + reload_interval=0) + assert ids(eng.search("first")) == ["a"] + state["blocks"] = [Block("b", "second")] + assert ids(eng.search("second")) == [] # token unchanged → no reload + state["token"] = 2 + assert ids(eng.search("second")) == ["b"] + + +def test_reload_interval_respected(): + calls = [] + eng = SearchEngine(lambda: [], reload_check=lambda: calls.append(1) or 1, reload_interval=3600) + eng.search("x") + eng.search("x") + assert len(calls) == 1 # only the initial check diff --git a/tests/test_markdown.py b/tests/test_markdown.py new file mode 100644 index 0000000..b4d3bb1 --- /dev/null +++ b/tests/test_markdown.py @@ -0,0 +1,119 @@ +import pytest + +from mdgrep.markdown import load_markdown, markdown_to_text, slugify, split_markdown + +DOC = """# Report +## Edition + +1. **[Rust 2.0 released](https://example.com/rust)** (100 points by alice) + + The Rust team announced a *major* release with `async` traits. + +2. **[Go generics](https://example.com/go)** (50 points by bob) + + Generics in Go are now __fast__. + +## Insights + +1. **Memory safety wins** + + - **Trend:** Rust adoption grows. + - **Why:** fewer bugs. + +2. **Typed languages** +lazy continuation line + +A closing paragraph +spanning two lines. + +--- + +*Footer text* + +``` +# not a heading +- not an item +``` +""" + + +def kinds(blocks): + return [(b.meta["kind"], b.meta["section"], b.line) for b in blocks] + + +def test_split_list(): + blocks = split_markdown(DOC, source="doc.md", split="list") + assert kinds(blocks) == [ + ("item", "Edition", 4), + ("item", "Edition", 8), + ("item", "Insights", 14), + ("item", "Insights", 19), + ("paragraph", "Insights", 22), + ("paragraph", "Insights", 27), + ("paragraph", "Insights", 29), + ] + first = blocks[0] + assert first.text == ("Rust 2.0 released (100 points by alice)\n\n" + "The Rust team announced a major release with async traits.") + assert first.title == "" + assert first.id == "doc.md:4" and first.source == "doc.md" + assert first.meta["markdown"].startswith("1. **[Rust 2.0 released]") + assert "Trend: Rust adoption grows." in blocks[2].text + assert "lazy continuation line" in blocks[3].text + assert "not a heading" in blocks[6].text + + +def test_split_heading(): + blocks = split_markdown(DOC, split="heading") + assert [(b.title, b.anchor) for b in blocks] == [ + ("Report", "report"), ("Edition", "edition"), ("Insights", "insights")] + assert "Go generics" in blocks[1].text + assert not blocks[1].text.startswith("Edition") + + +def test_split_paragraph(): + blocks = split_markdown(DOC, split="paragraph") + texts = [b.text.split("\n")[0] for b in blocks] + assert "Rust 2.0 released (100 points by alice)" in texts + assert "The Rust team announced a major release with async traits." in texts + assert "A closing paragraph" in texts + fence = [b for b in blocks if "not a heading" in b.text] + assert len(fence) == 1 and "not an item" in fence[0].text + + +def test_split_file(): + blocks = split_markdown(DOC, source="x.md", split="file") + assert len(blocks) == 1 and blocks[0].title == "Report" and blocks[0].line == 1 + + +def test_invalid_split_mode(): + with pytest.raises(ValueError): + split_markdown("x", split="bogus") + + +def test_markdown_to_text(): + md = ("- **Bold** and *italic* and __b2__ and `code` and ~~gone~~\n" + " [link](https://x.y/a_(b)) ![alt](img.png) \n" + "> quote with snake_case_name") + assert markdown_to_text(md) == ( + "Bold and italic and b2 and code and gone\n" + "link alt https://auto.link\n" + "quote with snake_case_name") + + +def test_slugify(): + assert slugify("AI/ML Insights & Trends") == "aiml-insights--trends" + assert slugify("Über **uns**") == "über-uns" + + +def test_load_markdown_directory_and_meta(tmp_path): + (tmp_path / "sub").mkdir() + (tmp_path / "a.md").write_text("first para\n\nsecond para\n", encoding="utf-8") + (tmp_path / "sub" / "b.md").write_text("- item one\n- item two\n", encoding="utf-8") + (tmp_path / "ignore.txt").write_text("nope", encoding="utf-8") + blocks = load_markdown(tmp_path, split="list", meta=lambda p: {"file": p.name}) + assert [(b.meta["file"], b.text) for b in blocks] == [ + ("a.md", "first para"), ("a.md", "second para"), + ("b.md", "item one"), ("b.md", "item two")] + rel = load_markdown(tmp_path, relative_to=tmp_path) + assert [b.id for b in rel] == ["a.md:1", "a.md:3", "sub/b.md:1"] diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..a7bdf6f --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,187 @@ +import time + +import pytest + +from mdgrep import QuerySyntaxError, QueryTimeout, compile_query + + +def m(query, text): + return compile_query(query).matches(text) + + +# ── Terms ──────────────────────────────────────────────────────────────── + +def test_substring_case_insensitive(): + assert m("rust", "The Rustacean book") + assert m("RUST", "trusted") + assert not m("rust", "golang") + + +def test_umlauts_case_insensitive(): + assert m("übersetzung", "Die ÜBERSETZUNG ist fertig") + assert m("ÄRGER", "kein ärger") + + +def test_phrase_matches_any_whitespace(): + assert m('"open source"', "Open\n Source software") + assert not m('"open source"', "open the source") + + +def test_phrase_escaped_quote(): + assert m(r'"say \"hi\""', 'they say "hi" loudly') + + +def test_regex(): + assert m(r"/llms?\b/", "Large LLMs today") + assert m(r"/gpt-\d+/", "GPT-5 release") + assert not m(r"/^foo$/", "foobar") + + +def test_regex_escaped_slash(): + assert m(r"/tcp\/ip/", "TCP/IP stack") + + +def test_words_with_special_characters(): + assert m("AT&T", "at&t merger") + assert m("TCP/IP", "the tcp/ip suite") + assert m("e-mail", "send an E-Mail") + + +def test_lowercase_keywords_are_terms(): + assert m("and", "this AND that") + assert not m("and", "nothing here") + + +# ── Boolean logic ──────────────────────────────────────────────────────── + +@pytest.mark.parametrize("query", ["a b", "a AND b", "a UND b", "a & b", "a && b"]) +def test_and_variants(query): + assert compile_query(query).matches("a b") + assert not compile_query(query).matches("a only") + + +@pytest.mark.parametrize("query", ["a OR b", "a ODER b", "a | b", "a || b"]) +def test_or_variants(query): + assert compile_query(query).matches("b only") + assert not compile_query(query).matches("nothing") + + +@pytest.mark.parametrize("query", ["NOT x", "NICHT x", "-x", "!x"]) +def test_not_variants(query): + assert compile_query(query).matches("abc") + assert not compile_query(query).matches("xyz") + + +def test_precedence_and_binds_tighter_than_or(): + q = compile_query("alpha OR beta AND gamma") # alpha OR (beta AND gamma) + assert q.matches("alpha") + assert not q.matches("beta") + assert q.matches("beta gamma") + + +def test_parentheses_override_precedence(): + q = compile_query("(alpha OR beta) AND gamma") + assert not q.matches("alpha") + assert q.matches("alpha gamma") + assert q.matches("beta gamma") + + +def test_not_binds_tighter_than_and(): + q = compile_query("NOT alpha beta") # (NOT alpha) AND beta + assert q.matches("beta") + assert not q.matches("alpha beta") + + +def test_nested_groups_and_negation(): + q = compile_query("(rust OR go) AND NOT (crypto OR /blockchain/)") + assert q.matches("Rust compiler") + assert not q.matches("Go crypto wallet") + assert not q.matches("rust blockchain") + + +def test_double_negation(): + assert m("NOT NOT foo", "foo") + + +# ── Spans (highlighting) ───────────────────────────────────────────────── + +def test_spans_merged_and_sorted(): + q = compile_query("open OR source") + assert q.spans("open source open") == [(0, 4), (5, 11), (12, 16)] + + +def test_spans_exclude_negated_terms(): + q = compile_query("rust -crypto") + assert q.spans("rust and crypto") == [(0, 4)] + + +def test_spans_double_negation_counts_as_positive(): + q = compile_query("NOT NOT foo") + assert q.spans("a foo") == [(2, 5)] + + +def test_spans_ignore_empty_regex_matches(): + q = compile_query("/x*/") + assert q.spans("abc") == [] + assert q.spans("axxb") == [(1, 3)] + + +def test_spans_overlapping_merged(): + q = compile_query("abc OR bcd") + assert q.spans("abcd") == [(0, 4)] + + +# ── Errors ─────────────────────────────────────────────────────────────── + +@pytest.mark.parametrize("query,code,position", [ + ("", "empty_query", 0), + (" ", "empty_query", 0), + ("(a OR b", "unclosed_paren", 0), + ("a OR b)", "unexpected_paren", 6), + ("a OR", "missing_operand", 4), + ("OR a", "missing_operand", 0), + ("a AND OR b", "missing_operand", 6), + ("()", "empty_group", 0), + ('"open', "unterminated_phrase", 0), + ("x /abc", "unterminated_regex", 2), + ("x /a(b/", "invalid_regex", 2), + ("NOT", "missing_operand", 3), +]) +def test_syntax_errors(query, code, position): + with pytest.raises(QuerySyntaxError) as exc: + compile_query(query) + assert exc.value.code == code + assert exc.value.position == position + assert exc.value.to_dict()["code"] == code + + +def test_limits(): + with pytest.raises(QuerySyntaxError) as exc: + compile_query("a" * 20, max_length=10) + assert exc.value.code == "too_long" + with pytest.raises(QuerySyntaxError) as exc: + compile_query("a b c d", max_terms=3) + assert exc.value.code == "too_many_terms" + + +# ── Timeouts (ReDoS protection) ────────────────────────────────────────── + +def test_catastrophic_regex_times_out(): + q = compile_query("/(a|aa)+$/") + text = "a" * 36 + "!" + start = time.monotonic() + with pytest.raises(QueryTimeout): + q.matches(text, timeout=0.2) + assert time.monotonic() - start < 2 + + +def test_spans_timeout(): + q = compile_query("/(a|aa)+$/") + with pytest.raises(QueryTimeout): + q.spans("a" * 36 + "!", timeout=0.2) + + +def test_expired_deadline_raises(): + q = compile_query("foo") + with pytest.raises(QueryTimeout): + q.matches("foo", deadline=time.monotonic() - 1) diff --git a/tests/test_snippets.py b/tests/test_snippets.py new file mode 100644 index 0000000..776cc87 --- /dev/null +++ b/tests/test_snippets.py @@ -0,0 +1,59 @@ +from mdgrep.snippets import ELLIPSIS, segments + + +def join(segs): + return "".join(s["t"] for s in segs) + + +def test_short_text_full_with_hits(): + segs = segments("hello world", [(6, 11)]) + assert segs == [{"t": "hello ", "hit": False}, {"t": "world", "hit": True}] + + +def test_no_spans_returns_whole_text(): + assert segments("abc", []) == [{"t": "abc", "hit": False}] + + +def test_long_text_windowed_around_hit(): + text = " ".join(f"w{i}" for i in range(400)) + hit_start = text.index("w200") + segs = segments(text, [(hit_start, hit_start + 4)], max_chars=100, context=30) + assert segs[0]["t"] == ELLIPSIS and segs[-1]["t"] == ELLIPSIS + assert any(s["hit"] and s["t"] == "w200" for s in segs) + assert len(join(segs)) < 150 + + +def test_windows_snap_to_word_boundaries(): + text = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda" + start = text.index("epsilon") + segs = segments(text, [(start, start + 7)], max_chars=20, context=8) + inner = join(s for s in segs if s["t"] != ELLIPSIS) + assert not inner.startswith("lta") and inner.split()[0] in ("gamma", "delta") + + +def test_multiple_hits_separate_windows(): + text = "x " * 300 + "foo " + "y " * 300 + "bar " + "z " * 300 + f, b = text.index("foo"), text.index("bar") + segs = segments(text, [(f, f + 3), (b, b + 3)], max_chars=200, context=20) + hits = [s["t"] for s in segs if s["hit"]] + assert hits == ["foo", "bar"] + assert sum(1 for s in segs if s["t"] == ELLIPSIS) == 3 + + +def test_budget_limits_windows_but_keeps_first_hit(): + text = ("filler " * 50 + "hit ") * 10 + spans = [] + pos = 0 + while (i := text.find("hit", pos)) != -1: + spans.append((i, i + 3)) + pos = i + 3 + segs = segments(text, spans, max_chars=60, context=20) + assert sum(1 for s in segs if s["hit"]) >= 1 + assert len(join(segs)) < 200 + + +def test_no_hits_long_text_shows_beginning(): + text = "start " + "word " * 200 + segs = segments(text, [], max_chars=50) + assert segs[0]["t"].startswith("start") + assert segs[-1]["t"] == ELLIPSIS diff --git a/tests/test_web.py b/tests/test_web.py new file mode 100644 index 0000000..5068596 --- /dev/null +++ b/tests/test_web.py @@ -0,0 +1,80 @@ +import pytest + +from mdgrep import Block, SearchEngine +from mdgrep.web import create_app + +BLOCKS = [ + Block("1", "rust news", title="Rust", meta={"lang": "en", "url": "u1", "markdown": "**rust**"}), + Block("2", "rust nachrichten", title="Rust", meta={"lang": "de", "url": "u1", "markdown": "x"}), + Block("3", "go news", title="Go", meta={"lang": "en", "url": "u2", "markdown": "y"}), + Block("4", "rust again", title="Rust", meta={"lang": "en", "url": "u1", "markdown": "z"}), +] + + +@pytest.fixture +def client(): + app = create_app(SearchEngine(BLOCKS), filters=["lang"], + group_by=lambda b: (b.meta["lang"], b.meta["url"])) + return app.test_client() + + +def test_search_ok(client): + data = client.get("/search?q=rust").get_json() + assert data["total_blocks"] == 3 and data["total_groups"] == 2 + first = data["results"][0] + assert first["block"]["id"] == "1" and [b["id"] for b in first["also"]] == ["4"] + assert "markdown" not in first["block"]["meta"] + assert "markdown" not in first["also"][0]["meta"] + + +def test_filters(client): + assert client.get("/search?q=rust&lang=de").get_json()["total_blocks"] == 1 + assert client.get("/search?q=rust&lang=all").get_json()["total_blocks"] == 3 + assert client.get("/search?q=rust&lang=de,en").get_json()["total_blocks"] == 3 + + +def test_paging_and_limit_clamping(client): + data = client.get("/search?q=news OR rust&limit=1").get_json() + assert len(data["results"]) == 1 and data["next_offset"] == 1 + data = client.get("/search?q=rust&limit=9999&offset=-5").get_json() + assert data["limit"] == 100 and data["offset"] == 0 + + +def test_empty_query(client): + data = client.get("/search?q=").get_json() + assert data["results"] == [] and data["total_groups"] == 0 + + +def test_syntax_error_400(client): + resp = client.get("/search?q=(rust") + assert resp.status_code == 400 + assert resp.get_json()["error"] == {"code": "unclosed_paren", + "message": "Missing closing parenthesis", "position": 0} + + +def test_too_long_query(client): + resp = client.get("/search?q=" + "a" * 400) + assert resp.status_code == 400 and resp.get_json()["error"]["code"] == "too_long" + + +def test_unicode_not_escaped(client): + app = create_app(SearchEngine([Block("x", "Größe")])) + body = app.test_client().get("/search?q=größe").get_data(as_text=True) + assert "Größe" in body + + +def test_widget_assets_served(client): + assert client.get("/mdgrep-static/mdgrep-widget.js").status_code == 200 + assert client.get("/mdgrep-static/mdgrep-widget.css").status_code == 200 + + +def test_static_dir_and_prefix(tmp_path): + (tmp_path / "index.html").write_text("

home

") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "index.html").write_text("sub page") + app = create_app(SearchEngine(BLOCKS), url_prefix="/api", static_dir=tmp_path) + c = app.test_client() + assert c.get("/").get_data(as_text=True) == "

home

" + assert c.get("/sub/").get_data(as_text=True) == "sub page" + assert c.get("/api/search?q=go").get_json()["total_blocks"] == 1 + assert c.get("/../etc/passwd").status_code == 404