478 lines
17 KiB
Python
478 lines
17 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""网站离线镜像爬虫（纯标准库版，无需 requests / beautifulsoup4）
|
||
用法:
|
||
python mirror_stdlib.py <起始URL> [最大页数]
|
||
示例:
|
||
python mirror_stdlib.py http://171.80.3.206:39897/
|
||
python mirror_stdlib.py https://example.com/ 5000
|
||
|
||
行为:
|
||
- 页面 BFS 爬取（仅目标域），统一存为 .html 便于 file:// 浏览
|
||
- 附件（pdf/zip 等）任何域名都下载
|
||
- CSS/JS/图片/字体等资源下载并把 HTML/CSS 内链接改写为本地相对路径
|
||
- 断点续传（crawl_manifest.json），可重复运行补漏
|
||
- 每请求间隔 0.3s，礼貌抓取
|
||
"""
|
||
import os, re, sys, json, time, hashlib, threading, gzip, zlib
|
||
from urllib.parse import urlparse, urljoin, urldefrag, unquote
|
||
from urllib.request import Request, urlopen
|
||
from urllib.error import HTTPError, URLError
|
||
from collections import deque
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
|
||
if len(sys.argv) < 2:
|
||
print(__doc__)
|
||
sys.exit(1)
|
||
|
||
TARGET = sys.argv[1].rstrip("/")
|
||
MAX_PAGES = int(sys.argv[2]) if len(sys.argv) > 2 else 3000
|
||
|
||
ROOT = os.path.dirname(os.path.abspath(__file__))
|
||
HOST = urlparse(TARGET).netloc
|
||
HOST_DIR = re.sub(r'[<>:"\\|?*]', "_", HOST) # Windows 目录名非法字符替换（端口冒号）
|
||
OUT = os.path.join(ROOT, "mirror", HOST_DIR)
|
||
MANIFEST = os.path.join(ROOT, "crawl_manifest.json")
|
||
QUEUE_FILE = os.path.join(ROOT, "queue.json")
|
||
LOG = open(os.path.join(ROOT, "crawl_log.txt"), "a", encoding="utf-8")
|
||
|
||
PAGE_HOSTS = {HOST.lower()}
|
||
START_URLS = [TARGET]
|
||
ATT_EXT = {
|
||
".pdf", ".zip", ".rar", ".7z", ".stp", ".step", ".dwg", ".dxf", ".igs", ".iges",
|
||
".stl", ".xls", ".xlsx", ".csv", ".doc", ".docx", ".ppt", ".pptx", ".tar", ".gz",
|
||
}
|
||
SKIP_QUERY_KEYS = {"cmsfkt", "print", "lang", "language", "sprache", "session", "cfid", "cftoken", "search", "q"}
|
||
SKIP_QUERY_VALS = {"watchlist", "nl_add", "logout", "login", "cart", "basket"}
|
||
DELAY = 0.3
|
||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
||
SKIP_ASSET_HOSTS = {"etracker.com", "google-analytics.com", "googletagmanager.com",
|
||
"doubleclick.net", "hotjar.com", "statcounter.com", "matomo.org",
|
||
"google.com", "facebook.com", "twitter.com", "linkedin.com"}
|
||
failed_logged = set()
|
||
LOCK = threading.RLock()
|
||
POOL = ThreadPoolExecutor(max_workers=4)
|
||
|
||
manifest = {}
|
||
if os.path.exists(MANIFEST):
|
||
try:
|
||
manifest = json.load(open(MANIFEST, encoding="utf-8"))
|
||
except Exception:
|
||
pass
|
||
visited = {u for u, v in manifest.items() if v.get("status") == "ok"}
|
||
failed = []
|
||
ext_html_seen = set()
|
||
|
||
def log(*a):
|
||
s = " ".join(str(x) for x in a)
|
||
print(s, flush=True)
|
||
with LOCK:
|
||
LOG.write(s + "\n"); LOG.flush()
|
||
|
||
def save_manifest():
|
||
with LOCK:
|
||
with open(MANIFEST, "w", encoding="utf-8") as f:
|
||
json.dump(manifest, f, ensure_ascii=False, indent=1)
|
||
|
||
def load_q():
|
||
try:
|
||
return json.load(open(QUEUE_FILE, encoding="utf-8"))
|
||
except Exception:
|
||
return []
|
||
|
||
def save_q(q):
|
||
with LOCK:
|
||
json.dump(list(q), open(QUEUE_FILE, "w", encoding="utf-8"))
|
||
|
||
def norm_url(u):
|
||
u = u.strip()
|
||
if not u or u.startswith(("javascript:", "mailto:", "tel:", "data:", "sms:", "about:", "#")):
|
||
return None
|
||
u, frag = urldefrag(u)
|
||
p = urlparse(u)
|
||
if p.scheme not in ("http", "https"):
|
||
return None
|
||
netloc = (p.netloc or "").lower() # 保留端口，不能 split(":") 去端口
|
||
path = p.path or "/"
|
||
qs = []
|
||
if p.query:
|
||
for kv in p.query.split("&"):
|
||
k = kv.split("=", 1)[0].lower()
|
||
v = kv.split("=", 1)[1].lower() if "=" in kv else ""
|
||
if k in SKIP_QUERY_KEYS or v in SKIP_QUERY_VALS:
|
||
continue
|
||
qs.append(kv)
|
||
query = "&".join(qs)
|
||
return f"{p.scheme}://{netloc}{path}" + (f"?{query}" if query else "")
|
||
|
||
def is_attachment(u):
|
||
p = urlparse(u)
|
||
path = unquote(p.path).lower()
|
||
return path.endswith(tuple(ATT_EXT))
|
||
|
||
STATIC_EXT = (".jpg", ".jpeg", ".png", ".gif", ".ico", ".svg", ".bmp", ".webp", ".tif", ".tiff")
|
||
|
||
def is_static(u):
|
||
return urlparse(u).path.lower().endswith(STATIC_EXT)
|
||
|
||
def skip_asset(u):
|
||
p = urlparse(u)
|
||
h = (p.netloc or "").lower().split(":")[0]
|
||
for bad in SKIP_ASSET_HOSTS:
|
||
if h == bad or h.endswith("." + bad):
|
||
return True
|
||
if not p.path or p.path == "/":
|
||
return True
|
||
return False
|
||
|
||
def is_junk_page(u):
|
||
p = urlparse(u)
|
||
path = p.path.lower()
|
||
return path.endswith((".css", ".js", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg",
|
||
".woff", ".woff2", ".ttf", ".webp", ".bin", ".map"))
|
||
|
||
def sanitize_seg(seg):
|
||
seg = re.sub(r'[<>:"\\|?*]', "_", seg)
|
||
seg = seg.strip(" .")
|
||
if not seg:
|
||
seg = "_"
|
||
if len(seg) > 90:
|
||
seg = seg[:80] + "_" + hashlib.md5(seg.encode()).hexdigest()[:8]
|
||
return seg
|
||
|
||
def local_path(u, is_page):
|
||
p = urlparse(u)
|
||
path = unquote(p.path)
|
||
if path.endswith("/"):
|
||
path += "index"
|
||
segs = [sanitize_seg(s) for s in path.split("/") if s]
|
||
if not segs:
|
||
segs = ["index"]
|
||
name = segs.pop()
|
||
if is_page:
|
||
name += ".html"
|
||
else:
|
||
if "." not in name or not re.search(r"\.[A-Za-z0-9]{1,6}$", name):
|
||
name += ".bin"
|
||
if p.query:
|
||
qs = sanitize_seg(unquote(p.query).replace("/", "_").replace("\\", "_"))[:120]
|
||
name = name.rsplit(".", 1)[0] + "@@" + qs + "." + name.rsplit(".", 1)[1]
|
||
return os.path.join(*segs, name)
|
||
|
||
def http_get(u, timeout=30):
|
||
"""urllib GET，返回 (bytes, content_type)。处理 gzip/deflate。"""
|
||
req = Request(u, headers={"User-Agent": UA, "Accept-Encoding": "identity"})
|
||
resp = urlopen(req, timeout=timeout)
|
||
data = resp.read()
|
||
headers = resp.headers
|
||
enc = (headers.get("Content-Encoding", "") or "").lower()
|
||
if enc == "gzip":
|
||
try:
|
||
data = gzip.decompress(data)
|
||
except Exception:
|
||
pass
|
||
elif enc == "deflate":
|
||
try:
|
||
data = zlib.decompress(data)
|
||
except Exception:
|
||
pass
|
||
ct = headers.get("Content-Type", "") or ""
|
||
return data, ct
|
||
|
||
def fetch(u, timeout=(15, 60)):
|
||
for attempt in range(3):
|
||
try:
|
||
return http_get(u, timeout=timeout[1])
|
||
except HTTPError as e:
|
||
if e.code in (400, 401, 403, 404, 410):
|
||
raise
|
||
if attempt == 2:
|
||
raise
|
||
time.sleep(2 * (attempt + 1))
|
||
except Exception:
|
||
if attempt == 2:
|
||
raise
|
||
time.sleep(2 * (attempt + 1))
|
||
|
||
def decode(data, ct):
|
||
"""字节 -> 文本，尽力识别编码。"""
|
||
enc = None
|
||
m = re.search(r'charset=["\']?([\w\-]+)', ct, re.I)
|
||
if m:
|
||
enc = m.group(1)
|
||
if not enc:
|
||
head = data[:4096]
|
||
m = re.search(br'<meta[^>]+charset=["\']?([\w\-]+)', head, re.I)
|
||
if m:
|
||
enc = m.group(1).decode("ascii", "ignore")
|
||
if not enc and data.startswith(b"\xef\xbb\xbf"):
|
||
enc = "utf-8-sig"
|
||
if not enc:
|
||
enc = "utf-8"
|
||
for e in (enc, "utf-8", "gbk", "gb18030", "latin-1"):
|
||
try:
|
||
return data.decode(e)
|
||
except Exception:
|
||
continue
|
||
return data.decode("latin-1", "replace")
|
||
|
||
def download_file(u, kind):
|
||
"""下载二进制资源/附件，返回相对路径。"""
|
||
if skip_asset(u):
|
||
return None
|
||
rel = local_path(u, is_page=False)
|
||
fp = os.path.join(OUT, rel)
|
||
with LOCK:
|
||
if u in manifest and manifest[u].get("status") == "ok" and os.path.exists(fp):
|
||
return rel
|
||
manifest[u] = {"path": rel, "type": kind, "status": "pending"}
|
||
save_manifest()
|
||
try:
|
||
data, ct = http_get(u)
|
||
os.makedirs(os.path.dirname(fp), exist_ok=True)
|
||
with open(fp, "wb") as f:
|
||
f.write(data)
|
||
# CSS: 改写 url() 引用为本地路径
|
||
if kind == "asset" and (rel.lower().endswith(".css") or "css" in ct):
|
||
try:
|
||
new_css = css_urls(data.decode("utf-8", "replace"), u)
|
||
with open(fp, "w", encoding="utf-8") as f:
|
||
f.write(new_css)
|
||
except Exception:
|
||
pass
|
||
with LOCK:
|
||
manifest[u] = {"path": rel, "type": kind, "status": "ok",
|
||
"size": len(data), "ct": ct}
|
||
save_manifest()
|
||
return rel
|
||
except HTTPError as e:
|
||
with LOCK:
|
||
failed.append((u, str(e)))
|
||
manifest[u] = {"path": rel, "type": kind,
|
||
"status": "dead" if e.code in (404, 410) else "failed", "err": str(e)[:200]}
|
||
save_manifest()
|
||
if u not in failed_logged:
|
||
failed_logged.add(u)
|
||
log(f" [FAIL] {kind} {u} : {e}")
|
||
return None
|
||
except Exception as e:
|
||
with LOCK:
|
||
failed.append((u, str(e)))
|
||
manifest[u] = {"path": rel, "type": kind, "status": "failed", "err": str(e)[:200]}
|
||
save_manifest()
|
||
if u not in failed_logged:
|
||
failed_logged.add(u)
|
||
log(f" [FAIL] {kind} {u} : {e}")
|
||
return None
|
||
|
||
def css_urls(css, css_url):
|
||
def rep(m):
|
||
raw = m.group(1).strip().strip("'\"")
|
||
if raw.startswith(("data:", "javascript:", "#")):
|
||
return m.group(0)
|
||
if raw.startswith("//"):
|
||
absu = "http:" + raw
|
||
else:
|
||
absu = urljoin(css_url, raw)
|
||
rel = download_file(absu, "asset")
|
||
if rel:
|
||
from_dir = os.path.dirname(local_path(css_url, is_page=False))
|
||
rr = os.path.relpath(rel, from_dir).replace("\\", "/")
|
||
return f"url({rr})"
|
||
return m.group(0)
|
||
return re.sub(r"url\(\s*(.*?)\s*\)", rep, css, flags=re.I | re.S)
|
||
|
||
def process_page(u, data, ct):
|
||
html = decode(data, ct)
|
||
page_rel = local_path(u, is_page=True)
|
||
page_dir = os.path.dirname(page_rel)
|
||
|
||
# base 标签：用于解析相对链接，随后删除
|
||
base_for_resolve = u
|
||
m = re.search(r'<base\b[^>]*href\s*=\s*(["\'])(.*?)\1', html, re.I | re.S)
|
||
if m:
|
||
base_for_resolve = urljoin(u, m.group(2).strip())
|
||
html = re.sub(r'<base\b[^>]*/?>', '', html, flags=re.I)
|
||
|
||
new_pages, atts, assets = [], [], []
|
||
|
||
def to_local(absu, is_page):
|
||
rel = local_path(absu, is_page=is_page)
|
||
return os.path.relpath(rel, page_dir).replace("\\", "/")
|
||
|
||
def handle_href(raw):
|
||
v = raw.strip()
|
||
if not v or v.startswith(("#", "javascript:", "mailto:", "tel:", "data:", "about:")):
|
||
return raw
|
||
frag = ""
|
||
if "#" in v:
|
||
v, _, fr = v.partition("#")
|
||
frag = "#" + fr
|
||
absu = norm_url(urljoin(base_for_resolve, v))
|
||
if not absu:
|
||
return raw
|
||
host = (urlparse(absu).netloc or "").lower()
|
||
if is_attachment(absu):
|
||
atts.append(absu); rel = local_path(absu, False)
|
||
elif is_static(absu):
|
||
assets.append(absu); rel = local_path(absu, False)
|
||
elif host in PAGE_HOSTS:
|
||
new_pages.append(absu); rel = local_path(absu, True)
|
||
else:
|
||
ext_html_seen.add(absu); return raw
|
||
return to_local(absu, is_page=not (is_attachment(absu) or is_static(absu))) + frag
|
||
|
||
def handle_src(raw):
|
||
v = raw.strip()
|
||
if not v or v.startswith(("data:", "javascript:", "about:")):
|
||
return raw
|
||
absu = norm_url(urljoin(base_for_resolve, v))
|
||
if not absu:
|
||
return raw
|
||
assets.append(absu)
|
||
return to_local(absu, False)
|
||
|
||
# href / src / data / poster 属性
|
||
def sub_attr(html, attr, handler):
|
||
pat = re.compile(r'\b(' + attr + r')\s*=\s*(["\'])(.*?)\2', re.I | re.S)
|
||
return pat.sub(lambda m: m.group(1) + "=" + m.group(2) + handler(m.group(3)) + m.group(2), html)
|
||
|
||
html = sub_attr(html, r'href', handle_href)
|
||
html = sub_attr(html, r'src', handle_src)
|
||
html = sub_attr(html, r'data', handle_src)
|
||
html = sub_attr(html, r'poster', handle_src)
|
||
|
||
# srcset（逗号分隔的 URL + 描述符列表）
|
||
def sub_srcset(html):
|
||
def rep(m):
|
||
parts = []
|
||
for part in m.group(3).split(","):
|
||
part = part.strip()
|
||
if not part:
|
||
continue
|
||
toks = part.split()
|
||
su = toks[0]
|
||
rest = " ".join(toks[1:])
|
||
newu = handle_src(su)
|
||
parts.append(newu + (" " + rest if rest else ""))
|
||
return m.group(1) + "=" + m.group(2) + ", ".join(parts) + m.group(2)
|
||
pat = re.compile(r'\b(srcset)\s*=\s*(["\'])(.*?)\2', re.I | re.S)
|
||
return pat.sub(rep, html)
|
||
html = sub_srcset(html)
|
||
|
||
# data-* 属性：仅提取 URL 用于发现页面，不改写（避免破坏 JS）
|
||
for m in re.finditer(r'\b(data-[a-z0-9_-]+)\s*=\s*(["\'])(.*?)\2', html, re.I | re.S):
|
||
v = m.group(3).strip()
|
||
if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(html?|php|cfm|aspx|jsp)\b", v)):
|
||
continue
|
||
absu = norm_url(urljoin(base_for_resolve, v))
|
||
if not absu:
|
||
continue
|
||
host = (urlparse(absu).netloc or "").lower()
|
||
if host in PAGE_HOSTS and not is_attachment(absu) and not is_junk_page(absu):
|
||
new_pages.append(absu)
|
||
|
||
# 确保有 charset meta
|
||
if not re.search(r'<meta[^>]+charset', html, re.I):
|
||
m = re.search(r'<head\b[^>]*>', html, re.I)
|
||
if m:
|
||
html = html[:m.end()] + '<meta charset="utf-8">' + html[m.end():]
|
||
|
||
fp = os.path.join(OUT, page_rel)
|
||
os.makedirs(os.path.dirname(fp), exist_ok=True)
|
||
with open(fp, "w", encoding="utf-8") as f:
|
||
f.write(html)
|
||
manifest[u] = {"path": page_rel, "type": "page", "status": "ok", "size": len(data)}
|
||
save_manifest()
|
||
return new_pages, atts, assets
|
||
|
||
def main():
|
||
q = deque()
|
||
for u in load_q() + [norm_url(u) for u in START_URLS] + \
|
||
[u for u, v in manifest.items() if v.get("type") == "page" and v.get("status") in ("failed", "pending")]:
|
||
u = norm_url(u)
|
||
if u and u not in q and not is_junk_page(u):
|
||
q.append(u)
|
||
log(f"队列 {len(q)} 个页面待处理")
|
||
pages_done = 0
|
||
|
||
pend = [(u, manifest[u].get("type", "asset")) for u, v in manifest.items()
|
||
if v.get("type") in ("asset", "attachment") and v.get("status") in ("pending", "failed")]
|
||
if pend:
|
||
log(f"续传未完成文件 {len(pend)} 个")
|
||
for u, k in pend:
|
||
POOL.submit(download_file, u, k)
|
||
|
||
while q:
|
||
u = q.popleft()
|
||
if u in visited:
|
||
continue
|
||
visited.add(u)
|
||
if is_junk_page(u):
|
||
manifest[u] = {"type": "page", "status": "dead", "err": "junk"}
|
||
continue
|
||
try:
|
||
data, ct = fetch(u)
|
||
except HTTPError as e:
|
||
failed.append((u, str(e)))
|
||
manifest[u] = {"type": "page", "status": "dead" if e.code in (404, 410) else "failed", "err": str(e)[:200]}
|
||
save_manifest()
|
||
if u not in failed_logged:
|
||
failed_logged.add(u)
|
||
log(f"[FAIL] page {u} : {e}")
|
||
time.sleep(DELAY)
|
||
continue
|
||
except Exception as e:
|
||
failed.append((u, str(e)))
|
||
manifest[u] = {"type": "page", "status": "failed", "err": str(e)[:200]}
|
||
save_manifest()
|
||
if u not in failed_logged:
|
||
failed_logged.add(u)
|
||
log(f"[FAIL] page {u} : {e}")
|
||
time.sleep(DELAY)
|
||
continue
|
||
|
||
pth = urlparse(u).path.lower()
|
||
if "html" in ct.lower() or pth.endswith((".htm", ".html", ".cfm", ".php", ".aspx", ".jsp")):
|
||
try:
|
||
new_pages, atts, assets = process_page(u, data, ct)
|
||
pages_done += 1
|
||
for a in atts:
|
||
POOL.submit(download_file, a, "attachment")
|
||
for a in assets:
|
||
POOL.submit(download_file, a, "asset")
|
||
for p in new_pages:
|
||
if p not in visited and p not in q:
|
||
q.append(p)
|
||
save_q(q)
|
||
if pages_done % 25 == 0:
|
||
log(f"... 已处理 {pages_done} 页, 队列 {len(q)}, 文件 {len(manifest)}")
|
||
except Exception as e:
|
||
log(f"[ERR] parse {u}: {e}")
|
||
else:
|
||
download_file(u, "attachment" if is_attachment(u) else "asset")
|
||
|
||
if pages_done > MAX_PAGES:
|
||
log("达到页面数上限，停止")
|
||
break
|
||
time.sleep(DELAY)
|
||
|
||
POOL.shutdown(wait=True)
|
||
log("=" * 60)
|
||
log(f"完成。页面 {pages_done} 个，manifest 条目 {len(manifest)} 个")
|
||
types = {}
|
||
for v in manifest.values():
|
||
types[v.get("type", "?")] = types.get(v.get("type", "?"), 0) + 1
|
||
log("类型统计:", types)
|
||
if failed:
|
||
log(f"失败 {len(failed)} 个:")
|
||
for u, e in failed:
|
||
log(" ", u, "|", e)
|
||
if ext_html_seen:
|
||
log(f"未跟进的站外 HTML 链接 {len(ext_html_seen)} 个 (示例):")
|
||
for u in list(ext_html_seen)[:10]:
|
||
log(" ", u)
|
||
|
||
if __name__ == "__main__":
|
||
log(f"=== 开始镜像 {TARGET} @ {time.strftime('%Y-%m-%d %H:%M:%S')} ===")
|
||
main()
|