555 lines
22 KiB
Python
555 lines
22 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""通用网站离线镜像爬虫（skill: website-mirror）
|
||
用法:
|
||
python mirror.py <起始URL> [额外页面域,逗号分隔] [最大页数]
|
||
示例:
|
||
python mirror.py https://zh.example.com/
|
||
python mirror.py https://www.example.com/ "media.example.com,static.example.com" 5000
|
||
|
||
行为:
|
||
- 页面 BFS 爬取（仅目标域 + 额外域），统一存为 .html 便于 file:// 浏览
|
||
- 附件（pdf/zip/CAD 等）任何域名都下载
|
||
- CSS/JS/图片/字体等页面资源下载并把 HTML/CSS 内链接改写为本地相对路径
|
||
- 断点续传（crawl_manifest.json），可重复运行补漏
|
||
- 每请求间隔 0.3s，单线程，礼貌抓取
|
||
输出:
|
||
mirror/<域名>/... 可离线浏览的站点副本
|
||
crawl_manifest.json 全部 URL 的下载记录
|
||
crawl_log.txt 日志与失败清单
|
||
"""
|
||
import os, re, sys, json, time, hashlib, warnings, threading
|
||
from urllib.parse import urlparse, urljoin, urldefrag, unquote, quote
|
||
from collections import deque
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
|
||
warnings.filterwarnings("ignore")
|
||
|
||
if len(sys.argv) < 2:
|
||
print(__doc__)
|
||
sys.exit(1)
|
||
|
||
TARGET = sys.argv[1].rstrip("/")
|
||
EXTRA_HOSTS = {h.strip().lower() for h in sys.argv[2].split(",") if h.strip()} if len(sys.argv) > 2 else set()
|
||
MAX_PAGES = int(sys.argv[3]) if len(sys.argv) > 3 else 3000
|
||
|
||
ROOT = os.path.dirname(os.path.abspath(__file__))
|
||
HOST = urlparse(TARGET).netloc
|
||
HOST_DIR = re.sub(r'[<>:"\\|?*]', "_", HOST) # Windows 目录名非法字符替换（端口冒号等）
|
||
OUT = os.path.join(ROOT, "mirror", HOST_DIR)
|
||
MANIFEST = os.path.join(ROOT, "crawl_manifest.json")
|
||
QUEUE_FILE = os.path.join(ROOT, "queue.json")
|
||
LOG = open(os.path.join(ROOT, "crawl_log.txt"), "a", encoding="utf-8")
|
||
|
||
PAGE_HOSTS = {HOST.lower()} | EXTRA_HOSTS
|
||
START_URLS = [TARGET]
|
||
ATT_EXT = {
|
||
".pdf", ".zip", ".rar", ".7z", ".stp", ".step", ".dwg", ".dxf", ".igs", ".iges",
|
||
".stl", ".easm", ".eprt", ".sldprt", ".sldasm", ".slddrw", ".xls", ".xlsx",
|
||
".csv", ".doc", ".docx", ".ppt", ".pptx", ".catpart", ".catproduct", ".3mf", ".jt",
|
||
".ifc", ".rfa", ".dgn", ".prt", ".asm", ".sdp", ".sat", ".stpz",
|
||
}
|
||
# 跳过的查询参数（表单/会话/语言切换等爬虫陷阱）
|
||
SKIP_QUERY_KEYS = {"cmsfkt", "print", "lang", "language", "sprache", "session", "cfid", "cftoken", "search", "q"}
|
||
SKIP_QUERY_VALS = {"watchlist", "nl_add", "logout", "login", "cart", "basket"}
|
||
DELAY = 0.3
|
||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
||
# 统计探针/广告等注定失败的域名，直接跳过
|
||
SKIP_ASSET_HOSTS = {"etracker.com", "google-analytics.com", "googletagmanager.com",
|
||
"doubleclick.net", "hotjar.com", "statcounter.com", "matomo.org"}
|
||
failed_logged = set()
|
||
LOCK = threading.RLock() # 可重入：持锁代码内还会调用 save_manifest/log
|
||
POOL = ThreadPoolExecutor(max_workers=4) # 文件下载并发线程池
|
||
|
||
sess = requests.Session()
|
||
sess.headers.update({"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"})
|
||
|
||
manifest = {}
|
||
if os.path.exists(MANIFEST):
|
||
try:
|
||
manifest = json.load(open(MANIFEST, encoding="utf-8"))
|
||
except Exception:
|
||
pass
|
||
visited = {u for u, v in manifest.items() if v.get("status") == "ok"}
|
||
failed = []
|
||
ext_html_seen = set()
|
||
|
||
def log(*a):
|
||
s = " ".join(str(x) for x in a)
|
||
print(s, flush=True)
|
||
with LOCK:
|
||
LOG.write(s + "\n"); LOG.flush()
|
||
|
||
def save_manifest():
|
||
with LOCK:
|
||
with open(MANIFEST, "w", encoding="utf-8") as f:
|
||
json.dump(manifest, f, ensure_ascii=False, indent=1)
|
||
|
||
def load_q():
|
||
"""恢复上次未处理完的页面队列。"""
|
||
try:
|
||
return json.load(open(QUEUE_FILE, encoding="utf-8"))
|
||
except Exception:
|
||
return []
|
||
|
||
def save_q(q):
|
||
with LOCK:
|
||
json.dump(list(q), open(QUEUE_FILE, "w", encoding="utf-8"))
|
||
|
||
def recover_queue():
|
||
"""从已保存页面的本地链接反向恢复站内页面 URL（弥补崩溃丢失的队列）。"""
|
||
out = set()
|
||
for u, v in manifest.items():
|
||
if v.get("type") != "page" or v.get("status") != "ok":
|
||
continue
|
||
fp = os.path.join(OUT, v.get("path", ""))
|
||
if not os.path.exists(fp):
|
||
continue
|
||
try:
|
||
html = open(fp, encoding="utf-8", errors="replace").read()
|
||
except Exception:
|
||
continue
|
||
for m in re.finditer(r'(?:href|src|data-link|data-href)\s*=\s*["\']([^"\']+)["\']', html, re.I):
|
||
h = m.group(1).strip()
|
||
if h.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
|
||
continue
|
||
cand = urljoin(u, h)
|
||
p = urlparse(cand)
|
||
if p.netloc.lower().split(":")[0] not in PAGE_HOSTS:
|
||
continue
|
||
if p.path.lower().endswith((".jpg", ".jpeg", ".png", ".gif", ".css", ".js",
|
||
".ico", ".svg", ".woff", ".woff2", ".ttf", ".webp")):
|
||
continue
|
||
path = p.path
|
||
if path.endswith(".html"):
|
||
path = path[:-5]
|
||
if path in ("/index", "/index.html"): # 首页别名归一
|
||
path = "/"
|
||
path, _, qs = path.partition("@@") # 反向还原查询参数编码（改写过的链接）
|
||
if qs:
|
||
path = path + "?" + qs
|
||
elif p.query: # 未改写的原始链接：保留 query
|
||
path = path + "?" + p.query
|
||
cand = norm_url(f"{p.scheme}://{p.netloc}{path}")
|
||
if cand and not is_attachment(cand) and not is_junk_page(cand):
|
||
out.add(cand)
|
||
return out
|
||
|
||
def norm_url(u):
|
||
u = u.strip()
|
||
if not u or u.startswith(("javascript:", "mailto:", "tel:", "data:", "sms:", "#")):
|
||
return None
|
||
u, frag = urldefrag(u)
|
||
p = urlparse(u)
|
||
if p.scheme not in ("http", "https"):
|
||
return None
|
||
host = p.netloc.lower().split(":")[0]
|
||
path = p.path or "/"
|
||
qs = []
|
||
if p.query:
|
||
for kv in p.query.split("&"):
|
||
k = kv.split("=", 1)[0].lower()
|
||
v = kv.split("=", 1)[1].lower() if "=" in kv else ""
|
||
if k in SKIP_QUERY_KEYS or v in SKIP_QUERY_VALS:
|
||
continue
|
||
qs.append(kv)
|
||
query = "&".join(qs)
|
||
return f"{p.scheme}://{host}{path}" + (f"?{query}" if query else "")
|
||
|
||
def is_attachment(u):
|
||
p = urlparse(u)
|
||
path = unquote(p.path).lower()
|
||
if path.endswith(tuple(ATT_EXT)):
|
||
return True
|
||
if "/d3/" in path and p.query: # 下载中心类动态链接
|
||
return True
|
||
return False
|
||
|
||
STATIC_EXT = (".jpg", ".jpeg", ".png", ".gif", ".ico", ".svg", ".bmp", ".webp", ".tif", ".tiff")
|
||
|
||
def is_static(u):
|
||
"""<a href> 直接指向图片等静态文件时按资源处理（不追加 .html）。"""
|
||
return urlparse(u).path.lower().endswith(STATIC_EXT)
|
||
|
||
def skip_asset(u):
|
||
p = urlparse(u)
|
||
h = p.netloc.lower().split(":")[0]
|
||
for bad in SKIP_ASSET_HOSTS: # 含子域名匹配
|
||
if h == bad or h.endswith("." + bad):
|
||
return True
|
||
if not p.path or p.path == "/":
|
||
return True
|
||
return False
|
||
|
||
def is_junk_page(u):
|
||
"""解析噪声/静态资源伪装成的页面 URL。"""
|
||
p = urlparse(u)
|
||
path = p.path.lower()
|
||
if path.endswith((".css", ".js", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg",
|
||
".woff", ".woff2", ".ttf", ".webp", ".bin")):
|
||
return True
|
||
if any(s in ("http", "https") for s in [x for x in p.path.split("/") if x]):
|
||
return True
|
||
return False
|
||
|
||
def sanitize_seg(seg):
|
||
seg = re.sub(r'[<>:"\\|?*]', "_", seg)
|
||
seg = seg.strip(" .")
|
||
if not seg:
|
||
seg = "_"
|
||
if len(seg) > 90:
|
||
seg = seg[:80] + "_" + hashlib.md5(seg.encode()).hexdigest()[:8]
|
||
return seg
|
||
|
||
def local_path(u, is_page):
|
||
"""URL → 本地相对路径。文件名使用真实 Unicode。"""
|
||
p = urlparse(u)
|
||
path = unquote(p.path)
|
||
if path.endswith("/"):
|
||
path += "index"
|
||
segs = [sanitize_seg(s) for s in path.split("/") if s]
|
||
if not segs:
|
||
segs = ["index"]
|
||
name = segs.pop()
|
||
if is_page:
|
||
name += ".html"
|
||
else:
|
||
if "." not in name or not re.search(r"\.[A-Za-z0-9]{1,6}$", name):
|
||
name += ".bin"
|
||
if p.query:
|
||
qs = sanitize_seg(unquote(p.query).replace("/", "_").replace("\\", "_"))[:120]
|
||
name = name.rsplit(".", 1)[0] + "@@" + qs + "." + name.rsplit(".", 1)[1]
|
||
return os.path.join(*segs, name)
|
||
|
||
def fetch(u, timeout=(15, 60)):
|
||
for attempt in range(3):
|
||
try:
|
||
r = sess.get(u, timeout=timeout)
|
||
r.raise_for_status()
|
||
return r
|
||
except requests.exceptions.HTTPError as e:
|
||
if e.response is not None and e.response.status_code in (400, 401, 403, 404, 410):
|
||
raise
|
||
if attempt == 2:
|
||
raise
|
||
time.sleep(2 * (attempt + 1))
|
||
except Exception:
|
||
if attempt == 2:
|
||
raise
|
||
time.sleep(2 * (attempt + 1))
|
||
|
||
def download_file(u, kind):
|
||
"""下载二进制资源/附件（流式+硬时限+并发安全），返回相对路径。"""
|
||
if skip_asset(u):
|
||
return None
|
||
rel = local_path(u, is_page=False)
|
||
fp = os.path.join(OUT, rel)
|
||
with LOCK:
|
||
if u in manifest and manifest[u].get("status") == "ok" and os.path.exists(fp):
|
||
return rel
|
||
manifest[u] = {"path": rel, "type": kind, "status": "pending"}
|
||
save_manifest()
|
||
try:
|
||
with requests.get(u, timeout=(15, 60), stream=True,
|
||
headers={"User-Agent": UA}) as r:
|
||
r.raise_for_status()
|
||
os.makedirs(os.path.dirname(fp), exist_ok=True)
|
||
deadline = time.time() + 600 # 单文件总时限 10 分钟，防慢速滴流卡死
|
||
total = 0
|
||
with open(fp, "wb") as f:
|
||
for chunk in r.iter_content(65536):
|
||
if time.time() > deadline:
|
||
raise TimeoutError("总时限到")
|
||
f.write(chunk)
|
||
total += len(chunk)
|
||
ct = r.headers.get("Content-Type", "")
|
||
# CSS: 下载其 url() 引用并改写为本地路径
|
||
if kind == "asset" and (rel.lower().endswith(".css") or "css" in ct):
|
||
try:
|
||
with open(fp, "rb") as f:
|
||
data = f.read()
|
||
new_css = css_urls(data.decode("utf-8", "replace"), u)
|
||
with open(fp, "w", encoding="utf-8") as f:
|
||
f.write(new_css)
|
||
except Exception:
|
||
pass
|
||
with LOCK:
|
||
manifest[u] = {"path": rel, "type": kind, "status": "ok",
|
||
"size": total, "ct": ct}
|
||
save_manifest()
|
||
return rel
|
||
except Exception as e:
|
||
with LOCK:
|
||
failed.append((u, str(e)))
|
||
dead = isinstance(e, requests.exceptions.HTTPError) and \
|
||
e.response is not None and e.response.status_code in (404, 410)
|
||
manifest[u] = {"path": rel, "type": kind,
|
||
"status": "dead" if dead else "failed", "err": str(e)[:200]}
|
||
save_manifest()
|
||
if u not in failed_logged:
|
||
failed_logged.add(u)
|
||
log(f" [FAIL] {kind} {u} : {e}")
|
||
return None
|
||
|
||
def css_urls(css, css_url):
|
||
def rep(m):
|
||
raw = m.group(1).strip().strip("'\"")
|
||
if raw.startswith(("data:", "http://", "https://", "//")):
|
||
absu = raw if raw.startswith(("http://", "https://")) else "https:" + raw
|
||
else:
|
||
absu = urljoin(css_url, raw)
|
||
if absu.startswith(("data:", "javascript:")):
|
||
return m.group(0)
|
||
rel = download_file(absu, "asset")
|
||
if rel:
|
||
from_dir = os.path.dirname(local_path(css_url, is_page=False))
|
||
rr = os.path.relpath(rel, from_dir).replace("\\", "/")
|
||
return f"url({rr})"
|
||
return m.group(0)
|
||
return re.sub(r"url\(\s*(.*?)\s*\)", rep, css, flags=re.I | re.S)
|
||
|
||
def process_page(u, r):
|
||
soup = BeautifulSoup(r.content, "html.parser")
|
||
base = u
|
||
b = soup.find("base", href=True)
|
||
if b:
|
||
base = urljoin(u, b["href"])
|
||
|
||
rel_page = local_path(u, is_page=True)
|
||
page_dir = os.path.dirname(rel_page)
|
||
|
||
def to_local(absu, kind="page"):
|
||
"""绝对 URL → 相对当前页面的本地路径。"""
|
||
if absu is None:
|
||
return None
|
||
rel = local_path(absu, is_page=(kind == "page"))
|
||
rp = os.path.relpath(rel, page_dir).replace("\\", "/")
|
||
return rp
|
||
|
||
new_pages, atts, assets = [], [], []
|
||
|
||
for a in soup.find_all("a", href=True):
|
||
h = a["href"].strip()
|
||
if h.startswith("#"):
|
||
continue
|
||
absu = norm_url(urljoin(base, h))
|
||
if absu is None:
|
||
continue
|
||
host = urlparse(absu).netloc.lower().split(":")[0]
|
||
if is_attachment(absu):
|
||
atts.append(absu)
|
||
elif is_static(absu):
|
||
assets.append(absu) # 图片直链按资源下载
|
||
elif host in PAGE_HOSTS:
|
||
new_pages.append(absu)
|
||
else:
|
||
ext_html_seen.add(absu)
|
||
|
||
# JS 跳转类链接：遍历全部 data-* 属性，值像 URL 的（以 / 开头、含 pid=/documentId= 等）都按链接处理
|
||
for tag in soup.find_all(True):
|
||
for attr, val in list(tag.attrs.items()):
|
||
if not (attr.startswith("data-") and isinstance(val, str)):
|
||
continue
|
||
v = val.strip()
|
||
if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(cfm|html|php|aspx)\b|pid=|documentId=", v)):
|
||
continue
|
||
absu = norm_url(urljoin(base, v))
|
||
if absu is None:
|
||
continue
|
||
host = urlparse(absu).netloc.lower().split(":")[0]
|
||
if is_attachment(absu):
|
||
atts.append(absu)
|
||
elif is_static(absu):
|
||
assets.append(absu)
|
||
elif host in PAGE_HOSTS:
|
||
new_pages.append(absu)
|
||
|
||
for tag in soup.find_all(["img", "script", "source", "embed", "video", "audio", "iframe"]):
|
||
for attr in ("src", "data", "poster"):
|
||
if tag.has_attr(attr):
|
||
absu = norm_url(urljoin(base, tag[attr].strip()))
|
||
if absu:
|
||
assets.append(absu)
|
||
if tag.has_attr("srcset"):
|
||
for part in tag["srcset"].split(","):
|
||
part = part.strip()
|
||
if part:
|
||
su = part.split()[0]
|
||
absu = norm_url(urljoin(base, su))
|
||
if absu:
|
||
assets.append(absu)
|
||
if tag.name == "iframe" and tag.has_attr("src"):
|
||
absu = norm_url(urljoin(base, tag["src"].strip()))
|
||
if absu and urlparse(absu).netloc.lower().split(":")[0] in PAGE_HOSTS and not is_attachment(absu):
|
||
new_pages.append(absu)
|
||
|
||
for l in soup.find_all("link", href=True):
|
||
rels = " ".join(l.get("rel", [])).lower()
|
||
if any(r in rels for r in ("preconnect", "dns-prefetch", "canonical", "alternate")):
|
||
continue # 预连接/SEO 标签不是真实资源
|
||
absu = norm_url(urljoin(base, l["href"].strip()))
|
||
if absu:
|
||
assets.append(absu)
|
||
|
||
# ---- 改写链接为本地路径 ----
|
||
for a in soup.find_all("a", href=True):
|
||
h = a["href"].strip()
|
||
if h.startswith(("#", "javascript:", "mailto:", "tel:")):
|
||
continue
|
||
absu = norm_url(urljoin(base, h))
|
||
if absu is None:
|
||
continue
|
||
frag = ""
|
||
if "#" in h:
|
||
frag = "#" + h.split("#", 1)[1]
|
||
host = urlparse(absu).netloc.lower().split(":")[0]
|
||
if is_attachment(absu) or is_static(absu):
|
||
a["href"] = (to_local(absu, "asset") or absu) + frag
|
||
elif host in PAGE_HOSTS:
|
||
a["href"] = to_local(absu, "page") + frag
|
||
|
||
for tag in soup.find_all(["img", "script", "source", "embed", "video", "audio"]):
|
||
for attr in ("src", "data", "poster"):
|
||
if tag.has_attr(attr):
|
||
absu = norm_url(urljoin(base, tag[attr].strip()))
|
||
if absu:
|
||
tag[attr] = to_local(absu, "asset") or absu
|
||
if tag.has_attr("srcset"):
|
||
parts = []
|
||
for part in tag["srcset"].split(","):
|
||
part = part.strip()
|
||
if not part:
|
||
continue
|
||
su, *rest = part.split()
|
||
absu = norm_url(urljoin(base, su))
|
||
parts.append((to_local(absu, "asset") or absu) + (" " + " ".join(rest) if rest else ""))
|
||
tag["srcset"] = ", ".join(parts)
|
||
|
||
for l in soup.find_all("link", href=True):
|
||
rels = " ".join(l.get("rel", [])).lower()
|
||
if any(r in rels for r in ("preconnect", "dns-prefetch", "canonical", "alternate")):
|
||
continue
|
||
absu = norm_url(urljoin(base, l["href"].strip()))
|
||
if absu:
|
||
l["href"] = to_local(absu, "asset") or absu
|
||
|
||
for tag in soup.find_all(True):
|
||
for attr, val in list(tag.attrs.items()):
|
||
if not (attr.startswith("data-") and isinstance(val, str)):
|
||
continue
|
||
v = val.strip()
|
||
if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(cfm|html|php|aspx)\b|pid=|documentId=", v)):
|
||
continue
|
||
absu = norm_url(urljoin(base, v))
|
||
if absu:
|
||
tag[attr] = to_local(absu, "page") or absu
|
||
|
||
for f in soup.find_all("iframe", src=True):
|
||
absu = norm_url(urljoin(base, f["src"].strip()))
|
||
if absu:
|
||
host = urlparse(absu).netloc.lower().split(":")[0]
|
||
f["src"] = to_local(absu, "page" if host in PAGE_HOSTS else "asset") or absu
|
||
|
||
if not soup.find("meta", attrs={"charset": True}):
|
||
hd = soup.find("head")
|
||
if hd:
|
||
hd.insert(0, BeautifulSoup('<meta charset="utf-8">', "html.parser").find("meta"))
|
||
|
||
# 删除 <base> 标签：改写后的相对链接按页面自身目录解析，base 会劫持解析基准导致离线浏览全部 404
|
||
for btag in soup.find_all("base"):
|
||
btag.decompose()
|
||
|
||
fp = os.path.join(OUT, rel_page)
|
||
os.makedirs(os.path.dirname(fp), exist_ok=True)
|
||
with open(fp, "w", encoding="utf-8") as f:
|
||
f.write(str(soup))
|
||
manifest[u] = {"path": rel_page, "type": "page", "status": "ok", "size": len(r.content)}
|
||
save_manifest()
|
||
return new_pages, atts, assets
|
||
|
||
def main():
|
||
# 恢复队列：上次未处理的 + 种子 + 失败页面 + 从已存页面反推的链接（断点续传）
|
||
q = deque()
|
||
for u in load_q() + [norm_url(u) for u in START_URLS] + \
|
||
[u for u, v in manifest.items() if v.get("type") == "page" and v.get("status") in ("failed", "pending")] + \
|
||
sorted(recover_queue()):
|
||
u = norm_url(u)
|
||
if u and u not in q and not is_junk_page(u):
|
||
q.append(u)
|
||
log(f"队列恢复 {len(q)} 个页面待处理")
|
||
pages_done = 0
|
||
# 先续传上次中断未完成的文件 + 重试网络错误失败的附件（死链除外）
|
||
pend = [(u, manifest[u].get("type", "asset")) for u, v in manifest.items()
|
||
if v.get("type") in ("asset", "attachment") and v.get("status") in ("pending", "failed")]
|
||
if pend:
|
||
log(f"续传未完成文件 {len(pend)} 个")
|
||
for u, k in pend:
|
||
POOL.submit(download_file, u, k)
|
||
while q:
|
||
u = q.popleft()
|
||
if u in visited:
|
||
continue
|
||
visited.add(u)
|
||
if is_junk_page(u):
|
||
manifest[u] = {"type": "page", "status": "dead", "err": "junk"}
|
||
continue
|
||
try:
|
||
r = fetch(u)
|
||
except Exception as e:
|
||
failed.append((u, str(e)))
|
||
# 404/410 为永久死链，标记 dead 不再重试；其它错误下次续传时重试
|
||
dead = isinstance(e, requests.exceptions.HTTPError) and \
|
||
e.response is not None and e.response.status_code in (404, 410)
|
||
manifest[u] = {"type": "page", "status": "dead" if dead else "failed", "err": str(e)[:200]}
|
||
save_manifest()
|
||
if not dead or u not in failed_logged:
|
||
failed_logged.add(u)
|
||
log(f"[FAIL] page {u} : {e}")
|
||
time.sleep(DELAY)
|
||
continue
|
||
ct = r.headers.get("Content-Type", "")
|
||
pth = urlparse(u).path.lower()
|
||
if "html" in ct.lower() or pth.endswith((".cfm", ".html", ".htm")):
|
||
try:
|
||
new_pages, atts, assets = process_page(u, r)
|
||
pages_done += 1
|
||
for a in atts:
|
||
POOL.submit(download_file, a, "attachment")
|
||
for a in assets:
|
||
POOL.submit(download_file, a, "asset")
|
||
for p in new_pages:
|
||
if p not in visited and p not in q:
|
||
q.append(p)
|
||
save_q(q) # 队列持久化，中断后可续爬
|
||
if pages_done % 25 == 0:
|
||
log(f"... 已处理 {pages_done} 页, 队列 {len(q)}, 文件总数 {len(manifest)}")
|
||
except Exception as e:
|
||
log(f"[ERR] parse {u}: {e}")
|
||
else:
|
||
download_file(u, "attachment" if is_attachment(u) else "asset")
|
||
if pages_done > MAX_PAGES:
|
||
log("达到页面数上限，停止")
|
||
break
|
||
time.sleep(DELAY)
|
||
|
||
POOL.shutdown(wait=True) # 等待所有文件下载完成
|
||
log("=" * 60)
|
||
log(f"完成。页面 {pages_done} 个，manifest 条目 {len(manifest)} 个")
|
||
types = {}
|
||
for v in manifest.values():
|
||
types[v.get("type", "?")] = types.get(v.get("type", "?"), 0) + 1
|
||
log("类型统计:", types)
|
||
if failed:
|
||
log(f"失败 {len(failed)} 个:")
|
||
for u, e in failed:
|
||
log(" ", u, "|", e)
|
||
if ext_html_seen:
|
||
log(f"未跟进的站外 HTML 链接 {len(ext_html_seen)} 个 (示例):")
|
||
for u in list(ext_html_seen)[:10]:
|
||
log(" ", u)
|
||
|
||
if __name__ == "__main__":
|
||
log(f"=== 开始镜像 {TARGET} @ {time.strftime('%Y-%m-%d %H:%M:%S')} ===")
|
||
main()
|