feat: add local server fallback to AI analyzer, include_paths support, config_path in reports
- ai_analyzer: OpenRouter primary + localhost:8001/:8002 as dev fallback - crawler: support include_paths for targeted crawling - __main__: pass config_path through report pipeline - tests: 266 tests passing
This commit is contained in:
parent
989f5a933f
commit
013cf794bc
13 changed files with 479 additions and 60 deletions
|
|
@ -124,6 +124,8 @@ def _extract_links(soup: BeautifulSoup, base_url: str) -> dict:
|
|||
"link_rel": [],
|
||||
"script": [],
|
||||
"img": [],
|
||||
"audio": [],
|
||||
"video": [],
|
||||
"iframe": [],
|
||||
"form": [],
|
||||
"meta_refresh": [],
|
||||
|
|
@ -162,6 +164,19 @@ def _extract_links(soup: BeautifulSoup, base_url: str) -> dict:
|
|||
u = strip_cache_params(u)
|
||||
result["img"].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all(["audio", "video"]):
|
||||
media_type = tag.name # "audio" or "video"
|
||||
if tag.get("src"):
|
||||
u = resolve(tag["src"])
|
||||
if u:
|
||||
u = strip_cache_params(u)
|
||||
result[media_type].append({"url": u, "class": classify(u)})
|
||||
for src_tag in tag.find_all("source", src=True):
|
||||
u = resolve(src_tag["src"])
|
||||
if u:
|
||||
u = strip_cache_params(u)
|
||||
result[media_type].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("iframe", src=True):
|
||||
u = resolve(tag["src"])
|
||||
if u:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue