From 2526ee6789619ffcfb619aa29f1c7f63170b25bf Mon Sep 17 00:00:00 2001 From: dschlueter Date: Tue, 6 Oct 2026 23:46:50 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20Sprachlabels=20f=C3=BCr=20Codebl=C3=B6c?= =?UTF-8?q?ke=20aus=20Inhalt=20und=20Buchkontext=20(R1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Gedruckte Zeilennummern ("00 #include …") werden für die Erkennung ignoriert, bleiben aber im Block stehen. - Python zusätzlich per ast.parse (Kontrollstrukturen, def/class, Import mit Code) und an >>>-Sitzungen erkannt; YAML zählt nicht. - Vordurchlauf bestimmt die Sprache des Buches aus den sicher erkannten Blöcken; passt sie, reichen für kurze Schnipsel schwächere Signale (detected_by: "book-context"). Programmausgaben bleiben ohne Label. Ohne Label: C-Buch 518 -> 132 von 574, Python-Buch 492 -> 326 von 747. Co-Authored-By: Claude Opus 5.5 --- BEDIENUNGSANLEITUNG.md | 5 +- CHANGELOG.md | 9 ++++ README.md | 11 ++-- epub2md/codeblocks.py | 111 +++++++++++++++++++++++++++++++++++++- epub2md/html_converter.py | 5 ++ epub2md/pipeline.py | 17 ++++++ tests/test_codeblocks.py | 54 +++++++++++++++++++ tests/test_qa2_open.py | 2 +- 8 files changed, 207 insertions(+), 7 deletions(-) diff --git a/BEDIENUNGSANLEITUNG.md b/BEDIENUNGSANLEITUNG.md index 02b51d5..f1e3639 100644 --- a/BEDIENUNGSANLEITUNG.md +++ b/BEDIENUNGSANLEITUNG.md @@ -163,8 +163,9 @@ python3 epub2md.py python-buch.epub output/python-buch ``` Programmcode bleibt zeichengenau erhalten, inklusive Einrückung. Die Programmiersprache wird nur -eingetragen, wenn sie eindeutig erkennbar ist. Ist das ganze Buch in **einer** Sprache geschrieben, -kannst du sie für alle übrigen Codeblöcke vorgeben: +eingetragen, wenn sie eindeutig erkennbar ist. Ist ein Buch erkennbar in **einer** Sprache geschrieben, +erhalten auch kurze Codeschnipsel dieses Label (etwa `char ch = 65;` in einem C-Buch). Programmausgaben +bleiben ohne Label. Willst du wirklich alle übrigen Blöcke kennzeichnen, gib die Sprache vor: ```bash python3 epub2md.py python-buch.epub output/python-buch --code-language python diff --git a/CHANGELOG.md b/CHANGELOG.md index cd1abd9..da79cba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,15 @@ Versionsnummern nach [Semantic Versioning](https://semver.org/lang/de/). Markdown-Validator, Wortvergleich gegen Datenverlust, Prüfung des Pandoc-Wegs für HTML, EPUB und PDF). Beschreibung in `tools/qa/README.md`; erzeugte Daten landen in `.qa/`. +### Verbessert + +- Sprachlabels für Codeblöcke (R1): Gedruckte Zeilennummern (`00 #include …`) verdecken die Sprache + nicht mehr. Python wird zusätzlich per `ast.parse` erkannt (Kontrollstrukturen, `def`/`class`, Import + mit Code) sowie an `>>>`-Sitzungen. Ein Vordurchlauf bestimmt die Sprache des Buches aus den sicher + erkannten Blöcken. Passt sie, reichen für kurze Schnipsel schwächere Signale (`detected_by: + "book-context"`); Programmausgaben bleiben ohne Label. Ohne Label waren vorher im C-Buch 518 von 574 + Blöcken, jetzt 132 (fast nur Ausgaben). Im Python-Buch waren es 492 von 747, jetzt 326 (Ausgaben). + ## [0.3.0] – 2026-10-06 ### Neu diff --git a/README.md b/README.md index 01b8e72..e4974ce 100644 --- a/README.md +++ b/README.md @@ -246,9 +246,14 @@ weil CommonMark keine `[^…]`-Syntax kennt. - **Syntax-Highlighting:** Farb-Spans, ``/`` in Listings und Zeilennummern werden entfernt. Einrückung, Tabs, `<`, `>`, `&`, `$`, `|`, Backslashes und Backticks bleiben exakt erhalten. Die Sprache kommt aus Klassen/Attributen (`language-python`, `data-lang`, …) oder starken Inhaltssignalen - (`#include`, `def …:`, Shebang, …). Ist sie unsicher, bleibt das Label leer. Bei einsprachigen - Büchern kann `--code-language python` (o. Ä.) alle übrigen Blöcke kennzeichnen. Achtung: Das betrifft - auch Programmausgaben und Shell-Befehle im Buch. + (`#include`, Shebang, `>>>`, Python-Code, der sich parsen lässt und `def`/`class`/Kontrollstrukturen + enthält, …). Gedruckte Zeilennummern als Text (`00 #include …`) stören die Erkennung nicht und bleiben + im Block stehen. Sind sich die sicher erkannten Blöcke einig (mindestens 5, davon 90 % eine Sprache), + genügen für die übrigen schwächere Signale: in einem Python-Buch Zuweisungen und Aufrufe, in einem + Buch mit C-artiger Sprache Anweisungen und Deklarationen mit `;` oder `{` (im Bericht + `detected_by: "book-context"`). Programmausgaben (Literale, Zahlen, Text) bleiben ohne Label. + `--code-language python` (o. Ä.) kennzeichnet alle übrigen Blöcke, auch Programmausgaben und + Shell-Befehle im Buch. - **CSS-Formatierung:** Viele Verlage setzen Hervorhebungen nur über Stylesheets. epub2md wertet die CSS-Dateien des Buches aus: Klassen-, Element- und ID-Selektoren, Nachfahren- und Kind-Kombinatoren, Spezifität und `style`-Attribute. `@media`-Blöcke und Selektoren mit Pseudoklassen werden ignoriert. diff --git a/epub2md/codeblocks.py b/epub2md/codeblocks.py index 0fcc883..7731a9b 100644 --- a/epub2md/codeblocks.py +++ b/epub2md/codeblocks.py @@ -2,8 +2,10 @@ from __future__ import annotations +import ast import json import re +import warnings from typing import Any from .utils import classes, is_element, local @@ -127,9 +129,114 @@ def detect_language(el: Any) -> tuple[str, str]: return "", "" +LINE_NUMBER_PREFIX = re.compile(r"^(\d{1,4})( +|$)") + + +def strip_line_numbers(code: str) -> str: + """Remove printed line numbers ("00 #include …") – for language detection only, the block keeps them. + + Applies when at least 80 % of the non-blank lines (at least two) start with consecutive numbers. + """ + lines = code.split("\n") + filled = [ln for ln in lines if ln.strip()] + matches = [LINE_NUMBER_PREFIX.match(ln) for ln in filled] + numbers = [int(m.group(1)) for m in matches if m] + if len(numbers) < 2 or len(numbers) < 0.8 * len(filled): + return code + if any(b != a + 1 for a, b in zip(numbers, numbers[1:])): + return code + width = min((len(m.group(2)) for m in matches if m and m.group(2)), default=0) + out = [] + for ln in lines: + m = LINE_NUMBER_PREFIX.match(ln) if ln.strip() else None + out.append(ln[len(m.group(1)) + min(width, len(m.group(2))):] if m else ln) + return "\n".join(out) + + +def _python_module(text: str) -> ast.Module | None: + """The parsed module, or None when the text is not valid Python (never executed, warnings suppressed).""" + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return ast.parse(text) + except (SyntaxError, ValueError, RecursionError, MemoryError): + return None + + +PY_COMPOUND = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.If, ast.For, ast.AsyncFor, ast.While, + ast.With, ast.AsyncWith, ast.Try) +PY_CODE_EXPR = (ast.Call, ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp, ast.Lambda, ast.Await) + + +def _python_strength(text: str) -> int: + """2 = surely Python (block statement, or import plus code), 1 = Python code (assignment, call, + comprehension …), 0 = not Python or only literals/names – which is what program output looks like.""" + if any(ln.rstrip().endswith((";", "{")) for ln in text.split("\n")): + return 0 + module = _python_module(text) + if module is None or not module.body: + return 0 + imports = sum(isinstance(n, (ast.Import, ast.ImportFrom)) for n in module.body) + code = 0 + for node in module.body: + if isinstance(node, PY_COMPOUND): + return 2 + if isinstance(node, ast.Expr): + value = node.value + if isinstance(value, ast.Constant) and isinstance(value.value, str): + continue # docstring + if not isinstance(value, PY_CODE_EXPR): + return 0 # a bare name, number or literal line is program output ("Jane", "42", "[1, 4]") + code += 1 + elif isinstance(node, ast.AnnAssign) and node.value is None: + pass # "name: value" is YAML or a Makefile rule as often as Python + elif not isinstance(node, (ast.Import, ast.ImportFrom, ast.Pass)): + code += 1 + if imports and code: + return 2 + return 1 if code else 0 + + +CURLY_STATEMENT = re.compile(r"[=(]|^\s*[A-Za-z_]\w*[\s*]+[A-Za-z_*]") # assignment/call or "type name" + + +def _curly_signal(text: str) -> bool: + """C-family syntax: a statement or declaration ending with ';' or '{', or a function body on one line.""" + lines = [re.sub(r"(^|\s)//.*$", "", ln).rstrip() for ln in text.split("\n")] + if any(ln.endswith((";", "{")) and CURLY_STATEMENT.search(ln) for ln in lines): + return True + return bool(re.search(r"\w\s*\([^)]*\)\s*\{.*\}", text)) + + +CURLY_LANGS = {"c", "cpp", "java", "csharp", "javascript", "typescript", "go", "rust", "php", "kotlin", "swift", "scala"} + + +def detect_language_from_book(code: str, book_language: str) -> str: + """Weak content signals, accepted only when they fit the book's dominant language.""" + if not book_language: + return "" + text = strip_line_numbers(code).strip() + if not text: + return "" + if book_language == "python" and _python_strength(text) >= 1: + return "python" + if book_language in CURLY_LANGS and _curly_signal(text): + return book_language + return "" + + +def dominant_language(languages: list[str]) -> str: + """The book's language when the reliably detected blocks agree (at least 5 blocks, 90 %).""" + detected = [lang for lang in languages if lang and lang != "text"] + if len(detected) < 5: + return "" + lang = max(set(detected), key=detected.count) + return lang if detected.count(lang) >= 0.9 * len(detected) else "" + + def detect_language_from_content(code: str) -> str: """Very conservative content heuristics – only strong signals.""" - text = code.strip() + text = strip_line_numbers(code).strip() if not text: return "" first = text.split("\n", 1)[0] @@ -167,6 +274,8 @@ def detect_language_from_content(code: str) -> str: c_like = sum(1 for ln in lines if ln.rstrip().endswith((";", "{"))) if py_signals >= 2 and c_like == 0: return "python" + if re.search(r"^>>> \S", text, re.M) or _python_strength(text) == 2: + return "python" if re.match(r"\s*(SELECT|INSERT\s+INTO|CREATE\s+(TABLE|INDEX|VIEW)|UPDATE\s+\w+\s+SET|DELETE\s+FROM)\b", text): return "sql" if re.match(r"(?m)^FROM\s+\S+", text) and re.search(r"(?m)^(RUN|CMD|COPY|ENTRYPOINT)\s", text): diff --git a/epub2md/html_converter.py b/epub2md/html_converter.py index 15ff259..b8de5ad 100644 --- a/epub2md/html_converter.py +++ b/epub2md/html_converter.py @@ -22,6 +22,7 @@ from .css import StyleSheet, emphasis, has_highlight, inline_style, style_props from .codeblocks import ( code_text, detect_language, + detect_language_from_book, detect_language_from_content, fence_for, inline_code, @@ -323,6 +324,7 @@ class Renderer: self.consumed_formula_images: set[str] = set() self.formulas: list[dict[str, str]] = [] self.code_blocks: list[dict[str, Any]] = [] + self.book_code_language = "" # dominant language of the reliably detected blocks (pre-scan) self.endnotes_file = "" # per chapter state self.ch: Chapter | None = None @@ -799,6 +801,9 @@ class Renderer: if not lang: lang = detect_language_from_content(code) how = "content" if lang else "" + if not lang: + lang = detect_language_from_book(code, self.book_code_language) + how = "book-context" if lang else "" if not lang and self.opt.code_language: lang, how = self.opt.code_language, "option" if self.ch: diff --git a/epub2md/pipeline.py b/epub2md/pipeline.py index 45500bc..db5eae3 100644 --- a/epub2md/pipeline.py +++ b/epub2md/pipeline.py @@ -17,6 +17,8 @@ from urllib.parse import unquote from . import TOOL_NAME, __version__ from .chapters import SPECIAL_KINDS, ChapterBuilder, heading_level +from .codeblocks import (code_text, detect_language, detect_language_from_content, dominant_language, + is_block_code_container, join_wrap_markers) from .diagnostics import Diagnostics from .css import StyleSheet, base_style, emphasis from .epub_reader import EpubReader, decode_markup, parse_xml @@ -292,6 +294,7 @@ def _render(reader: EpubReader, package: Any, chapters: list[Chapter], resolver: notes_title = "Anmerkungen" if language.startswith("de") else "Notes" endnotes_number = 999 if len(chapters) < 999 else len(chapters) + 1 renderer.endnotes_file = f"chapters/{endnotes_number:0{width}d}-{slugify(notes_title, opts.slug_language)}.md" + renderer.book_code_language = _book_code_language(chapters) for ch in chapters: diag.step(f"Rendere {ch.filename}") ch.markdown = renderer.render_chapter(ch) @@ -312,6 +315,20 @@ def _render(reader: EpubReader, package: Any, chapters: list[Chapter], resolver: return _Rendered(renderer, media, cover_out, endnotes, endnotes_md, skipped) +def _book_code_language(chapters: list[Chapter]) -> str: + """Pre-scan: the language of the book's code blocks, from the reliably detected blocks only.""" + langs: list[str] = [] + for ch in chapters: + for _, item in ch.items: + if not is_element(item): + continue + for el in iter_elements(item): + if local(el.tag) in ("pre", "listing", "xmp", "plaintext") or is_block_code_container(el): + lang, _ = detect_language(el) + langs.append(lang or detect_language_from_content(join_wrap_markers(code_text(el))[0])) + return dominant_language(langs) + + def _suggest_code_language(renderer: Renderer, diag: Diagnostics) -> None: """Book-level hint: many unlabeled blocks while every recognized block has the same language.""" blocks = renderer.code_blocks diff --git a/tests/test_codeblocks.py b/tests/test_codeblocks.py index 0cf0ca9..6084d45 100644 --- a/tests/test_codeblocks.py +++ b/tests/test_codeblocks.py @@ -118,3 +118,57 @@ def test_print_wrap_markers_can_be_kept(convert): run = convert(Book(docs=[Doc("c.xhtml", body)]), "--keep-wrap-markers") (_, _, code), = fenced_blocks(run.chapter("001")) assert code == "a = (1,\n↪ 2)" + + +def test_printed_line_numbers_do_not_hide_the_language(convert): + listing = ("00 // Kapitel1/hallo.c\n01 #include <stdio.h>\n \n02 int main(void) {\n" + "03 printf(\"Hallo\\n\");\n04 return 0;\n05 }") + run = convert(Book(docs=[Doc("c.xhtml", f"

N

{listing}
")])) + (_, lang, code), = fenced_blocks(run.chapter("001")) + assert lang == "c" + assert code.startswith("00 // Kapitel1/hallo.c\n01 #include ") # numbers stay in the block + + +def test_python_is_recognized_by_parsing(convert): + blocks = ["from collections import ChainMap\n\nchain = ChainMap({}, {})\nprint(chain)", + "zahl = -1\nif zahl < 0:\n print(\"negativ\")", + ">>> 2 + 3\n5", + "name: epub2md\nversion: 1", # YAML is valid Python syntax, but no Python code + "[1, 4, 9]", # program output + "x = compute()"] # Python, but too weak without book context + body = "

P

" + "".join(f"
{b}
" for b in blocks) + run = convert(Book(docs=[Doc("c.xhtml", body)])) + langs = [lang for _, lang, _ in fenced_blocks(run.chapter("001"))] + assert langs == ["python", "python", "python", "", "", ""] + + +def test_weak_signals_follow_the_book_language(convert): + c_listing = "#include <stdio.h>\nint main(void) { return 0; }" + snippets = ["char ch = 65; // 'A'", "if (a || b) {\n x();\n}", "int remove(const char *pathname);", + "struct node {\n int value;\n};", "Ergebnis: 12345", "|00003.14|", "gcc -Wall -o prog prog.c", + "Dezimal: 65; Zeichen: A"] + body = "

C

" + "".join(f"
{c_listing}
" for _ in range(5)) + body += "".join(f"
{s}
" for s in snippets) + run = convert(Book(docs=[Doc("c.xhtml", body)])) + langs = [lang for _, lang, _ in fenced_blocks(run.chapter("001"))][5:] + assert langs == ["c", "c", "c", "c", "", "", "", ""] + by = [item["detected_by"] for item in run.report["code_blocks"]["items"]][5:9] + assert by == ["book-context"] * 4 + + +def test_python_book_labels_assignments_but_not_output(convert): + py = "def f(x):\n return x" + snippets = ["foo = decorate(foo)", "random.sample(population, k)", "[x * x for x in range(5)]", + "True", "('A', 'B', 'C')", "Hallo Welt", "4\n3\nnohtyP", "Jane\n42\nPerson(name='Jane')"] + body = "

P

" + "".join(f"
{py}
" for _ in range(5)) + body += "".join(f"
{s}
" for s in snippets) + run = convert(Book(docs=[Doc("c.xhtml", body)])) + langs = [lang for _, lang, _ in fenced_blocks(run.chapter("001"))][5:] + assert langs == ["python", "python", "python", "", "", "", "", ""] + + +def test_mixed_book_gets_no_context_labels(convert): + body = "

M

" + "
def f(x):\n    return x
" * 3 + body += "
#include <stdio.h>\nint main(void) { return 0; }
" * 3 + "
x = compute();
" + run = convert(Book(docs=[Doc("c.xhtml", body)])) + assert fenced_blocks(run.chapter("001"))[-1][1] == "" diff --git a/tests/test_qa2_open.py b/tests/test_qa2_open.py index e9b1e41..7b003c2 100644 --- a/tests/test_qa2_open.py +++ b/tests/test_qa2_open.py @@ -39,7 +39,7 @@ def test_qa2_10_book_readme_recommends_pdf_via_html(convert): def test_qa2_11_code_language_hint(convert): py = "def f(x):\n return x\n\nclass A:\n pass" - body = "

C

" + "".join(f"
{py}\n# {i}
print({i})
" for i in range(10)) + body = "

C

" + "".join(f"
{py}\n# {i}
Ergebnis: {i}
" for i in range(10)) run = convert(Book(docs=[Doc("c.xhtml", body)])) hints = [i for i in run.report["infos"] if i["code"] == "code-language-hint"] assert hints and "--code-language python" in hints[0]["message"]