dschlueter
2526ee6789
- Gedruckte Zeilennummern ("00 #include …") werden für die Erkennung
ignoriert, bleiben aber im Block stehen.
- Python zusätzlich per ast.parse (Kontrollstrukturen, def/class, Import
mit Code) und an >>>-Sitzungen erkannt; YAML zählt nicht.
- Vordurchlauf bestimmt die Sprache des Buches aus den sicher erkannten
Blöcken; passt sie, reichen für kurze Schnipsel schwächere Signale
(detected_by: "book-context"). Programmausgaben bleiben ohne Label.
Ohne Label: C-Buch 518 -> 132 von 574, Python-Buch 492 -> 326 von 747.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
325 lines
13 KiB
Python
325 lines
13 KiB
Python
"""Code block detection, exact text extraction and language recognition."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import ast
|
||
import json
|
||
import re
|
||
import warnings
|
||
from typing import Any
|
||
|
||
from .utils import classes, is_element, local
|
||
|
||
LANG_ALIASES = {
|
||
"python": "python", "py": "python", "python3": "python", "py3": "python",
|
||
"bash": "bash", "shell": "bash", "sh": "sh", "zsh": "zsh",
|
||
"c": "c", "h": "c", "cpp": "cpp", "c++": "cpp", "cxx": "cpp", "hpp": "cpp",
|
||
"java": "java", "javascript": "javascript", "js": "javascript", "jsx": "javascript",
|
||
"typescript": "typescript", "ts": "typescript", "tsx": "typescript",
|
||
"json": "json", "yaml": "yaml", "yml": "yaml", "toml": "toml", "xml": "xml",
|
||
"html": "html", "xhtml": "html", "htm": "html", "css": "css", "sql": "sql",
|
||
"dockerfile": "dockerfile", "docker": "dockerfile", "makefile": "makefile", "make": "makefile",
|
||
"ini": "ini", "cfg": "ini", "conf": "ini", "text": "text", "plaintext": "text", "txt": "text",
|
||
"plain": "text", "rust": "rust", "rs": "rust", "go": "go", "golang": "go", "ruby": "ruby", "rb": "ruby",
|
||
"php": "php", "perl": "perl", "r": "r", "matlab": "matlab", "kotlin": "kotlin", "swift": "swift",
|
||
"scala": "scala", "haskell": "haskell", "csharp": "csharp", "cs": "csharp", "c#": "csharp",
|
||
"powershell": "powershell", "ps1": "powershell", "lua": "lua", "diff": "diff", "latex": "latex",
|
||
"tex": "latex", "markdown": "markdown", "md": "markdown", "graphql": "graphql", "nginx": "nginx",
|
||
}
|
||
BLOCK_CODE_CLASS = re.compile(
|
||
r"(programlisting|sourcecode|source-code|codeblock|code-block|codelisting|code_listing|^listing$|"
|
||
r"^code$|^pre$|^src$|^screen$|^verbatim$|^highlight$|^sourcecode$|^literallayout$)"
|
||
)
|
||
INLINE_CODE_CLASS = re.compile(
|
||
r"^(tt|code|mono|monospace|literal|pre|inlinecode|inline-code|computeroutput|userinput|"
|
||
r"varname|function|command|filename|classname|methodname|parameter)$|nonproportional|monospace"
|
||
)
|
||
LINE_NUMBER_CLASS = re.compile(r"^(lineno|linenos|linenumber|line-number|line-numbers|ln|gutter|lnt|linenodiv)$")
|
||
PRESENTATION_TAGS = {"button"}
|
||
BLOCK_CHILD_TAGS = {"p", "div", "table", "ul", "ol", "dl", "blockquote", "h1", "h2", "h3", "h4", "h5", "h6", "figure", "section"}
|
||
|
||
|
||
def is_block_code_container(el: Any) -> bool:
|
||
"""div/p whose class marks a code listing and which contains only inline content."""
|
||
if local(el.tag) not in ("div", "p"):
|
||
return False
|
||
if not any(BLOCK_CODE_CLASS.search(c) for c in classes(el)):
|
||
return False
|
||
for e in el.iter():
|
||
if e is el or not is_element(e):
|
||
continue
|
||
if local(e.tag) in BLOCK_CHILD_TAGS or local(e.tag) == "pre":
|
||
return False
|
||
return True
|
||
|
||
|
||
def is_inline_code(el: Any) -> bool:
|
||
name = local(el.tag)
|
||
if name in ("code", "samp", "tt", "kbd", "var"):
|
||
return name != "kbd" and name != "var"
|
||
return name == "span" and any(INLINE_CODE_CLASS.search(c) for c in classes(el))
|
||
|
||
|
||
def code_text(el: Any) -> str:
|
||
"""Exact text of a code element: <br> become newlines, line numbers/buttons are dropped."""
|
||
parts: list[str] = []
|
||
|
||
def walk(node: Any) -> None:
|
||
if node.text:
|
||
parts.append(node.text)
|
||
for child in node:
|
||
if is_element(child):
|
||
name = local(child.tag)
|
||
if name == "br":
|
||
parts.append("\n")
|
||
elif name in PRESENTATION_TAGS or any(LINE_NUMBER_CLASS.match(c) for c in classes(child)):
|
||
pass
|
||
elif name in ("p", "div") and parts and not "".join(parts).endswith("\n"):
|
||
parts.append("\n")
|
||
walk(child)
|
||
else:
|
||
walk(child)
|
||
if child.tail:
|
||
parts.append(child.tail)
|
||
|
||
walk(el)
|
||
text = "".join(parts).replace("\r\n", "\n").replace("\r", "\n").replace("\xa0", " ")
|
||
if text.startswith("\n"):
|
||
text = text[1:] # HTML ignores the first newline in <pre>
|
||
lines = text.split("\n")
|
||
while lines and not lines[-1].strip():
|
||
lines.pop()
|
||
while lines and not lines[0].strip():
|
||
lines.pop(0)
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _lang_from_value(value: str) -> str | None:
|
||
value = value.strip().lower()
|
||
for pattern in (r"(?:^|\s)(?:language|lang)-([\w+#-]+)", r"brush:\s*([\w+#-]+)", r"(?:^|\s)sourcecode\s+([\w+#-]+)"):
|
||
m = re.search(pattern, value)
|
||
if m and m.group(1) in LANG_ALIASES:
|
||
return LANG_ALIASES[m.group(1)]
|
||
return None
|
||
|
||
|
||
def detect_language(el: Any) -> tuple[str, str]:
|
||
"""Return (language, how). Empty language when not sure."""
|
||
candidates = [el]
|
||
parent = el.getparent()
|
||
for _ in range(2):
|
||
if parent is None:
|
||
break
|
||
candidates.append(parent)
|
||
parent = parent.getparent()
|
||
candidates.extend(e for e in el.iter() if is_element(e) and local(e.tag) == "code")
|
||
for cand in candidates:
|
||
for attr in ("data-language", "data-lang", "lang", "language", "data-code-language"):
|
||
value = (cand.get(attr) or "").strip().lower()
|
||
if value in LANG_ALIASES and attr != "lang":
|
||
return LANG_ALIASES[value], f"attribute {attr}"
|
||
cls = cand.get("class") or ""
|
||
lang = _lang_from_value(cls)
|
||
if lang:
|
||
return lang, "class"
|
||
if local(cand.tag) in ("pre", "code"):
|
||
for c in classes(cand):
|
||
if c in LANG_ALIASES and c not in ("text", "h", "r", "cs", "ts", "md", "rs"):
|
||
return LANG_ALIASES[c], "class"
|
||
return "", ""
|
||
|
||
|
||
LINE_NUMBER_PREFIX = re.compile(r"^(\d{1,4})( +|$)")
|
||
|
||
|
||
def strip_line_numbers(code: str) -> str:
|
||
"""Remove printed line numbers ("00 #include …") – for language detection only, the block keeps them.
|
||
|
||
Applies when at least 80 % of the non-blank lines (at least two) start with consecutive numbers.
|
||
"""
|
||
lines = code.split("\n")
|
||
filled = [ln for ln in lines if ln.strip()]
|
||
matches = [LINE_NUMBER_PREFIX.match(ln) for ln in filled]
|
||
numbers = [int(m.group(1)) for m in matches if m]
|
||
if len(numbers) < 2 or len(numbers) < 0.8 * len(filled):
|
||
return code
|
||
if any(b != a + 1 for a, b in zip(numbers, numbers[1:])):
|
||
return code
|
||
width = min((len(m.group(2)) for m in matches if m and m.group(2)), default=0)
|
||
out = []
|
||
for ln in lines:
|
||
m = LINE_NUMBER_PREFIX.match(ln) if ln.strip() else None
|
||
out.append(ln[len(m.group(1)) + min(width, len(m.group(2))):] if m else ln)
|
||
return "\n".join(out)
|
||
|
||
|
||
def _python_module(text: str) -> ast.Module | None:
|
||
"""The parsed module, or None when the text is not valid Python (never executed, warnings suppressed)."""
|
||
try:
|
||
with warnings.catch_warnings():
|
||
warnings.simplefilter("ignore")
|
||
return ast.parse(text)
|
||
except (SyntaxError, ValueError, RecursionError, MemoryError):
|
||
return None
|
||
|
||
|
||
PY_COMPOUND = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.If, ast.For, ast.AsyncFor, ast.While,
|
||
ast.With, ast.AsyncWith, ast.Try)
|
||
PY_CODE_EXPR = (ast.Call, ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp, ast.Lambda, ast.Await)
|
||
|
||
|
||
def _python_strength(text: str) -> int:
|
||
"""2 = surely Python (block statement, or import plus code), 1 = Python code (assignment, call,
|
||
comprehension …), 0 = not Python or only literals/names – which is what program output looks like."""
|
||
if any(ln.rstrip().endswith((";", "{")) for ln in text.split("\n")):
|
||
return 0
|
||
module = _python_module(text)
|
||
if module is None or not module.body:
|
||
return 0
|
||
imports = sum(isinstance(n, (ast.Import, ast.ImportFrom)) for n in module.body)
|
||
code = 0
|
||
for node in module.body:
|
||
if isinstance(node, PY_COMPOUND):
|
||
return 2
|
||
if isinstance(node, ast.Expr):
|
||
value = node.value
|
||
if isinstance(value, ast.Constant) and isinstance(value.value, str):
|
||
continue # docstring
|
||
if not isinstance(value, PY_CODE_EXPR):
|
||
return 0 # a bare name, number or literal line is program output ("Jane", "42", "[1, 4]")
|
||
code += 1
|
||
elif isinstance(node, ast.AnnAssign) and node.value is None:
|
||
pass # "name: value" is YAML or a Makefile rule as often as Python
|
||
elif not isinstance(node, (ast.Import, ast.ImportFrom, ast.Pass)):
|
||
code += 1
|
||
if imports and code:
|
||
return 2
|
||
return 1 if code else 0
|
||
|
||
|
||
CURLY_STATEMENT = re.compile(r"[=(]|^\s*[A-Za-z_]\w*[\s*]+[A-Za-z_*]") # assignment/call or "type name"
|
||
|
||
|
||
def _curly_signal(text: str) -> bool:
|
||
"""C-family syntax: a statement or declaration ending with ';' or '{', or a function body on one line."""
|
||
lines = [re.sub(r"(^|\s)//.*$", "", ln).rstrip() for ln in text.split("\n")]
|
||
if any(ln.endswith((";", "{")) and CURLY_STATEMENT.search(ln) for ln in lines):
|
||
return True
|
||
return bool(re.search(r"\w\s*\([^)]*\)\s*\{.*\}", text))
|
||
|
||
|
||
CURLY_LANGS = {"c", "cpp", "java", "csharp", "javascript", "typescript", "go", "rust", "php", "kotlin", "swift", "scala"}
|
||
|
||
|
||
def detect_language_from_book(code: str, book_language: str) -> str:
|
||
"""Weak content signals, accepted only when they fit the book's dominant language."""
|
||
if not book_language:
|
||
return ""
|
||
text = strip_line_numbers(code).strip()
|
||
if not text:
|
||
return ""
|
||
if book_language == "python" and _python_strength(text) >= 1:
|
||
return "python"
|
||
if book_language in CURLY_LANGS and _curly_signal(text):
|
||
return book_language
|
||
return ""
|
||
|
||
|
||
def dominant_language(languages: list[str]) -> str:
|
||
"""The book's language when the reliably detected blocks agree (at least 5 blocks, 90 %)."""
|
||
detected = [lang for lang in languages if lang and lang != "text"]
|
||
if len(detected) < 5:
|
||
return ""
|
||
lang = max(set(detected), key=detected.count)
|
||
return lang if detected.count(lang) >= 0.9 * len(detected) else ""
|
||
|
||
|
||
def detect_language_from_content(code: str) -> str:
|
||
"""Very conservative content heuristics – only strong signals."""
|
||
text = strip_line_numbers(code).strip()
|
||
if not text:
|
||
return ""
|
||
first = text.split("\n", 1)[0]
|
||
if first.startswith("#!"):
|
||
if re.search(r"\b(bash)\b", first):
|
||
return "bash"
|
||
if re.search(r"\b(sh|dash)\b", first):
|
||
return "sh"
|
||
if "zsh" in first:
|
||
return "zsh"
|
||
if "python" in first:
|
||
return "python"
|
||
if text.startswith("<?xml"):
|
||
return "xml"
|
||
if re.match(r"(?i)<!doctype html|<html[\s>]", text):
|
||
return "html"
|
||
lines = text.split("\n")
|
||
if re.search(r"^\s*#\s*include\s*[<\"]", text, re.M):
|
||
if re.search(r"#include\s*<(iostream|vector|string|map|memory|algorithm)>|std::|\bcout\b|\bnamespace\b", text):
|
||
return "cpp"
|
||
return "c"
|
||
if re.search(r"^\s*(int|void)\s+main\s*\(", text, re.M) and ";" in text:
|
||
return "c"
|
||
if re.search(r"^\s*public\s+(static\s+)?(class|interface)\s+\w+", text, re.M) and ";" in text:
|
||
return "java"
|
||
py_signals = 0
|
||
if re.search(r"^\s*def \w+\s*\(.*\)\s*(->\s*[^:]+)?:\s*(#.*)?$", text, re.M):
|
||
py_signals += 2
|
||
if re.search(r"^\s*class \w+(\(.*\))?:\s*$", text, re.M):
|
||
py_signals += 2
|
||
if re.search(r"^\s*(from [\w.]+ import [\w*., ]+|import [\w.]+( as \w+)?)\s*$", text, re.M):
|
||
py_signals += 1
|
||
if re.search(r"^\s*(for|while|if|elif|with|try|except)\b.*:\s*$", text, re.M):
|
||
py_signals += 1
|
||
c_like = sum(1 for ln in lines if ln.rstrip().endswith((";", "{")))
|
||
if py_signals >= 2 and c_like == 0:
|
||
return "python"
|
||
if re.search(r"^>>> \S", text, re.M) or _python_strength(text) == 2:
|
||
return "python"
|
||
if re.match(r"\s*(SELECT|INSERT\s+INTO|CREATE\s+(TABLE|INDEX|VIEW)|UPDATE\s+\w+\s+SET|DELETE\s+FROM)\b", text):
|
||
return "sql"
|
||
if re.match(r"(?m)^FROM\s+\S+", text) and re.search(r"(?m)^(RUN|CMD|COPY|ENTRYPOINT)\s", text):
|
||
return "dockerfile"
|
||
if text[0] in "{[" and len(lines) > 1:
|
||
try:
|
||
json.loads(text)
|
||
return "json"
|
||
except ValueError:
|
||
pass
|
||
return ""
|
||
|
||
|
||
WRAP_MARKER = re.compile(r"^[ \t]*[↪➥][ \t]?")
|
||
|
||
|
||
def join_wrap_markers(code: str) -> tuple[str, int]:
|
||
"""Undo print line wrapping: a line starting with ↪/➥ continues the previous line.
|
||
|
||
Publishers wrap over-long code lines at a space and mark the continuation; the logical line is
|
||
restored with exactly one space. Returns (code, number of joined lines).
|
||
"""
|
||
out: list[str] = []
|
||
joined = 0
|
||
for line in code.split("\n"):
|
||
m = WRAP_MARKER.match(line)
|
||
if m and out:
|
||
out[-1] = out[-1].rstrip(" \t") + " " + line[m.end():].lstrip(" \t")
|
||
joined += 1
|
||
else:
|
||
out.append(line)
|
||
return "\n".join(out), joined
|
||
|
||
|
||
def fence_for(code: str) -> str:
|
||
longest = max((len(m) for m in re.findall(r"`+", code)), default=0)
|
||
return "`" * max(3, longest + 1)
|
||
|
||
|
||
def inline_code(text: str) -> str:
|
||
text = re.sub(r"[ \t\r\n\f]+", " ", text)
|
||
if not text.strip():
|
||
return ""
|
||
longest = max((len(m) for m in re.findall(r"`+", text)), default=0)
|
||
ticks = "`" * (longest + 1)
|
||
pad = " " if text.startswith("`") or text.endswith("`") else ""
|
||
return f"{ticks}{pad}{text}{pad}{ticks}"
|