dschlueter
6db834addd
Fortsetzungszeilen werden wieder mit genau einem Leerzeichen zur ursprünglichen Codezeile zusammengefügt. Im Python-Beispielbuch sind das 195 Stellen; danach lassen sich 247 statt 203 von 255 Python-Blöcken parsen. Abschaltbar mit --keep-wrap-markers; der Bericht enthält wrap_lines_joined pro Block und einen Hinweis pro Kapitel. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
216 lines
8.7 KiB
Python
216 lines
8.7 KiB
Python
"""Code block detection, exact text extraction and language recognition."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from typing import Any
|
||
|
||
from .utils import classes, is_element, local
|
||
|
||
LANG_ALIASES = {
|
||
"python": "python", "py": "python", "python3": "python", "py3": "python",
|
||
"bash": "bash", "shell": "bash", "sh": "sh", "zsh": "zsh",
|
||
"c": "c", "h": "c", "cpp": "cpp", "c++": "cpp", "cxx": "cpp", "hpp": "cpp",
|
||
"java": "java", "javascript": "javascript", "js": "javascript", "jsx": "javascript",
|
||
"typescript": "typescript", "ts": "typescript", "tsx": "typescript",
|
||
"json": "json", "yaml": "yaml", "yml": "yaml", "toml": "toml", "xml": "xml",
|
||
"html": "html", "xhtml": "html", "htm": "html", "css": "css", "sql": "sql",
|
||
"dockerfile": "dockerfile", "docker": "dockerfile", "makefile": "makefile", "make": "makefile",
|
||
"ini": "ini", "cfg": "ini", "conf": "ini", "text": "text", "plaintext": "text", "txt": "text",
|
||
"plain": "text", "rust": "rust", "rs": "rust", "go": "go", "golang": "go", "ruby": "ruby", "rb": "ruby",
|
||
"php": "php", "perl": "perl", "r": "r", "matlab": "matlab", "kotlin": "kotlin", "swift": "swift",
|
||
"scala": "scala", "haskell": "haskell", "csharp": "csharp", "cs": "csharp", "c#": "csharp",
|
||
"powershell": "powershell", "ps1": "powershell", "lua": "lua", "diff": "diff", "latex": "latex",
|
||
"tex": "latex", "markdown": "markdown", "md": "markdown", "graphql": "graphql", "nginx": "nginx",
|
||
}
|
||
BLOCK_CODE_CLASS = re.compile(
|
||
r"(programlisting|sourcecode|source-code|codeblock|code-block|codelisting|code_listing|^listing$|"
|
||
r"^code$|^pre$|^src$|^screen$|^verbatim$|^highlight$|^sourcecode$|^literallayout$)"
|
||
)
|
||
INLINE_CODE_CLASS = re.compile(
|
||
r"^(tt|code|mono|monospace|literal|pre|inlinecode|inline-code|computeroutput|userinput|"
|
||
r"varname|function|command|filename|classname|methodname|parameter)$|nonproportional|monospace"
|
||
)
|
||
LINE_NUMBER_CLASS = re.compile(r"^(lineno|linenos|linenumber|line-number|line-numbers|ln|gutter|lnt|linenodiv)$")
|
||
PRESENTATION_TAGS = {"button"}
|
||
BLOCK_CHILD_TAGS = {"p", "div", "table", "ul", "ol", "dl", "blockquote", "h1", "h2", "h3", "h4", "h5", "h6", "figure", "section"}
|
||
|
||
|
||
def is_block_code_container(el: Any) -> bool:
|
||
"""div/p whose class marks a code listing and which contains only inline content."""
|
||
if local(el.tag) not in ("div", "p"):
|
||
return False
|
||
if not any(BLOCK_CODE_CLASS.search(c) for c in classes(el)):
|
||
return False
|
||
for e in el.iter():
|
||
if e is el or not is_element(e):
|
||
continue
|
||
if local(e.tag) in BLOCK_CHILD_TAGS or local(e.tag) == "pre":
|
||
return False
|
||
return True
|
||
|
||
|
||
def is_inline_code(el: Any) -> bool:
|
||
name = local(el.tag)
|
||
if name in ("code", "samp", "tt", "kbd", "var"):
|
||
return name != "kbd" and name != "var"
|
||
return name == "span" and any(INLINE_CODE_CLASS.search(c) for c in classes(el))
|
||
|
||
|
||
def code_text(el: Any) -> str:
|
||
"""Exact text of a code element: <br> become newlines, line numbers/buttons are dropped."""
|
||
parts: list[str] = []
|
||
|
||
def walk(node: Any) -> None:
|
||
if node.text:
|
||
parts.append(node.text)
|
||
for child in node:
|
||
if is_element(child):
|
||
name = local(child.tag)
|
||
if name == "br":
|
||
parts.append("\n")
|
||
elif name in PRESENTATION_TAGS or any(LINE_NUMBER_CLASS.match(c) for c in classes(child)):
|
||
pass
|
||
elif name in ("p", "div") and parts and not "".join(parts).endswith("\n"):
|
||
parts.append("\n")
|
||
walk(child)
|
||
else:
|
||
walk(child)
|
||
if child.tail:
|
||
parts.append(child.tail)
|
||
|
||
walk(el)
|
||
text = "".join(parts).replace("\r\n", "\n").replace("\r", "\n").replace("\xa0", " ")
|
||
if text.startswith("\n"):
|
||
text = text[1:] # HTML ignores the first newline in <pre>
|
||
lines = text.split("\n")
|
||
while lines and not lines[-1].strip():
|
||
lines.pop()
|
||
while lines and not lines[0].strip():
|
||
lines.pop(0)
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _lang_from_value(value: str) -> str | None:
|
||
value = value.strip().lower()
|
||
for pattern in (r"(?:^|\s)(?:language|lang)-([\w+#-]+)", r"brush:\s*([\w+#-]+)", r"(?:^|\s)sourcecode\s+([\w+#-]+)"):
|
||
m = re.search(pattern, value)
|
||
if m and m.group(1) in LANG_ALIASES:
|
||
return LANG_ALIASES[m.group(1)]
|
||
return None
|
||
|
||
|
||
def detect_language(el: Any) -> tuple[str, str]:
|
||
"""Return (language, how). Empty language when not sure."""
|
||
candidates = [el]
|
||
parent = el.getparent()
|
||
for _ in range(2):
|
||
if parent is None:
|
||
break
|
||
candidates.append(parent)
|
||
parent = parent.getparent()
|
||
candidates.extend(e for e in el.iter() if is_element(e) and local(e.tag) == "code")
|
||
for cand in candidates:
|
||
for attr in ("data-language", "data-lang", "lang", "language", "data-code-language"):
|
||
value = (cand.get(attr) or "").strip().lower()
|
||
if value in LANG_ALIASES and attr != "lang":
|
||
return LANG_ALIASES[value], f"attribute {attr}"
|
||
cls = cand.get("class") or ""
|
||
lang = _lang_from_value(cls)
|
||
if lang:
|
||
return lang, "class"
|
||
if local(cand.tag) in ("pre", "code"):
|
||
for c in classes(cand):
|
||
if c in LANG_ALIASES and c not in ("text", "h", "r", "cs", "ts", "md", "rs"):
|
||
return LANG_ALIASES[c], "class"
|
||
return "", ""
|
||
|
||
|
||
def detect_language_from_content(code: str) -> str:
|
||
"""Very conservative content heuristics – only strong signals."""
|
||
text = code.strip()
|
||
if not text:
|
||
return ""
|
||
first = text.split("\n", 1)[0]
|
||
if first.startswith("#!"):
|
||
if re.search(r"\b(bash)\b", first):
|
||
return "bash"
|
||
if re.search(r"\b(sh|dash)\b", first):
|
||
return "sh"
|
||
if "zsh" in first:
|
||
return "zsh"
|
||
if "python" in first:
|
||
return "python"
|
||
if text.startswith("<?xml"):
|
||
return "xml"
|
||
if re.match(r"(?i)<!doctype html|<html[\s>]", text):
|
||
return "html"
|
||
lines = text.split("\n")
|
||
if re.search(r"^\s*#\s*include\s*[<\"]", text, re.M):
|
||
if re.search(r"#include\s*<(iostream|vector|string|map|memory|algorithm)>|std::|\bcout\b|\bnamespace\b", text):
|
||
return "cpp"
|
||
return "c"
|
||
if re.search(r"^\s*(int|void)\s+main\s*\(", text, re.M) and ";" in text:
|
||
return "c"
|
||
if re.search(r"^\s*public\s+(static\s+)?(class|interface)\s+\w+", text, re.M) and ";" in text:
|
||
return "java"
|
||
py_signals = 0
|
||
if re.search(r"^\s*def \w+\s*\(.*\)\s*(->\s*[^:]+)?:\s*(#.*)?$", text, re.M):
|
||
py_signals += 2
|
||
if re.search(r"^\s*class \w+(\(.*\))?:\s*$", text, re.M):
|
||
py_signals += 2
|
||
if re.search(r"^\s*(from [\w.]+ import [\w*., ]+|import [\w.]+( as \w+)?)\s*$", text, re.M):
|
||
py_signals += 1
|
||
if re.search(r"^\s*(for|while|if|elif|with|try|except)\b.*:\s*$", text, re.M):
|
||
py_signals += 1
|
||
c_like = sum(1 for ln in lines if ln.rstrip().endswith((";", "{")))
|
||
if py_signals >= 2 and c_like == 0:
|
||
return "python"
|
||
if re.match(r"\s*(SELECT|INSERT\s+INTO|CREATE\s+(TABLE|INDEX|VIEW)|UPDATE\s+\w+\s+SET|DELETE\s+FROM)\b", text):
|
||
return "sql"
|
||
if re.match(r"(?m)^FROM\s+\S+", text) and re.search(r"(?m)^(RUN|CMD|COPY|ENTRYPOINT)\s", text):
|
||
return "dockerfile"
|
||
if text[0] in "{[" and len(lines) > 1:
|
||
try:
|
||
json.loads(text)
|
||
return "json"
|
||
except ValueError:
|
||
pass
|
||
return ""
|
||
|
||
|
||
WRAP_MARKER = re.compile(r"^[ \t]*[↪➥][ \t]?")
|
||
|
||
|
||
def join_wrap_markers(code: str) -> tuple[str, int]:
|
||
"""Undo print line wrapping: a line starting with ↪/➥ continues the previous line.
|
||
|
||
Publishers wrap over-long code lines at a space and mark the continuation; the logical line is
|
||
restored with exactly one space. Returns (code, number of joined lines).
|
||
"""
|
||
out: list[str] = []
|
||
joined = 0
|
||
for line in code.split("\n"):
|
||
m = WRAP_MARKER.match(line)
|
||
if m and out:
|
||
out[-1] = out[-1].rstrip(" \t") + " " + line[m.end():].lstrip(" \t")
|
||
joined += 1
|
||
else:
|
||
out.append(line)
|
||
return "\n".join(out), joined
|
||
|
||
|
||
def fence_for(code: str) -> str:
|
||
longest = max((len(m) for m in re.findall(r"`+", code)), default=0)
|
||
return "`" * max(3, longest + 1)
|
||
|
||
|
||
def inline_code(text: str) -> str:
|
||
text = re.sub(r"[ \t\r\n\f]+", " ", text)
|
||
if not text.strip():
|
||
return ""
|
||
longest = max((len(m) for m in re.findall(r"`+", text)), default=0)
|
||
ticks = "`" * (longest + 1)
|
||
pad = " " if text.startswith("`") or text.endswith("`") else ""
|
||
return f"{ticks}{pad}{text}{pad}{ticks}"
|