首次提交: OnebotCatalog 项目代码与文档（含 NX 按需生成服务二期、后台、一键启动）

2026-09-03 17:55:45 +08:00
commit fafa86d3a6
241 changed files with 78656 additions and 0 deletions
--- a/blog_mirror/mirror.py
+++ b/blog_mirror/mirror.py
@@ -0,0 +1,554 @@
+# -*- coding: utf-8 -*-
+"""通用网站离线镜像爬虫（skill: website-mirror）
+用法:
+    python mirror.py <起始URL> [额外页面域,逗号分隔] [最大页数]
+示例:
+    python mirror.py https://zh.example.com/
+    python mirror.py https://www.example.com/ "media.example.com,static.example.com" 5000
+
+行为:
+    - 页面 BFS 爬取（仅目标域 + 额外域），统一存为 .html 便于 file:// 浏览
+    - 附件（pdf/zip/CAD 等）任何域名都下载
+    - CSS/JS/图片/字体等页面资源下载并把 HTML/CSS 内链接改写为本地相对路径
+    - 断点续传（crawl_manifest.json），可重复运行补漏
+    - 每请求间隔 0.3s，单线程，礼貌抓取
+输出:
+    mirror/<域名>/...  可离线浏览的站点副本
+    crawl_manifest.json 全部 URL 的下载记录
+    crawl_log.txt       日志与失败清单
+"""
+import os, re, sys, json, time, hashlib, warnings, threading
+from urllib.parse import urlparse, urljoin, urldefrag, unquote, quote
+from collections import deque
+from concurrent.futures import ThreadPoolExecutor
+import requests
+from bs4 import BeautifulSoup
+
+warnings.filterwarnings("ignore")
+
+if len(sys.argv) < 2:
+    print(__doc__)
+    sys.exit(1)
+
+TARGET = sys.argv[1].rstrip("/")
+EXTRA_HOSTS = {h.strip().lower() for h in sys.argv[2].split(",") if h.strip()} if len(sys.argv) > 2 else set()
+MAX_PAGES = int(sys.argv[3]) if len(sys.argv) > 3 else 3000
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+HOST = urlparse(TARGET).netloc
+HOST_DIR = re.sub(r'[<>:"\\|?*]', "_", HOST)   # Windows 目录名非法字符替换（端口冒号等）
+OUT = os.path.join(ROOT, "mirror", HOST_DIR)
+MANIFEST = os.path.join(ROOT, "crawl_manifest.json")
+QUEUE_FILE = os.path.join(ROOT, "queue.json")
+LOG = open(os.path.join(ROOT, "crawl_log.txt"), "a", encoding="utf-8")
+
+PAGE_HOSTS = {HOST.lower()} | EXTRA_HOSTS
+START_URLS = [TARGET]
+ATT_EXT = {
+    ".pdf", ".zip", ".rar", ".7z", ".stp", ".step", ".dwg", ".dxf", ".igs", ".iges",
+    ".stl", ".easm", ".eprt", ".sldprt", ".sldasm", ".slddrw", ".xls", ".xlsx",
+    ".csv", ".doc", ".docx", ".ppt", ".pptx", ".catpart", ".catproduct", ".3mf", ".jt",
+    ".ifc", ".rfa", ".dgn", ".prt", ".asm", ".sdp", ".sat", ".stpz",
+}
+# 跳过的查询参数（表单/会话/语言切换等爬虫陷阱）
+SKIP_QUERY_KEYS = {"cmsfkt", "print", "lang", "language", "sprache", "session", "cfid", "cftoken", "search", "q"}
+SKIP_QUERY_VALS = {"watchlist", "nl_add", "logout", "login", "cart", "basket"}
+DELAY = 0.3
+UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+# 统计探针/广告等注定失败的域名，直接跳过
+SKIP_ASSET_HOSTS = {"etracker.com", "google-analytics.com", "googletagmanager.com",
+                    "doubleclick.net", "hotjar.com", "statcounter.com", "matomo.org"}
+failed_logged = set()
+LOCK = threading.RLock()   # 可重入：持锁代码内还会调用 save_manifest/log
+POOL = ThreadPoolExecutor(max_workers=4)   # 文件下载并发线程池
+
+sess = requests.Session()
+sess.headers.update({"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"})
+
+manifest = {}
+if os.path.exists(MANIFEST):
+    try:
+        manifest = json.load(open(MANIFEST, encoding="utf-8"))
+    except Exception:
+        pass
+visited = {u for u, v in manifest.items() if v.get("status") == "ok"}
+failed = []
+ext_html_seen = set()
+
+def log(*a):
+    s = " ".join(str(x) for x in a)
+    print(s, flush=True)
+    with LOCK:
+        LOG.write(s + "\n"); LOG.flush()
+
+def save_manifest():
+    with LOCK:
+        with open(MANIFEST, "w", encoding="utf-8") as f:
+            json.dump(manifest, f, ensure_ascii=False, indent=1)
+
+def load_q():
+    """恢复上次未处理完的页面队列。"""
+    try:
+        return json.load(open(QUEUE_FILE, encoding="utf-8"))
+    except Exception:
+        return []
+
+def save_q(q):
+    with LOCK:
+        json.dump(list(q), open(QUEUE_FILE, "w", encoding="utf-8"))
+
+def recover_queue():
+    """从已保存页面的本地链接反向恢复站内页面 URL（弥补崩溃丢失的队列）。"""
+    out = set()
+    for u, v in manifest.items():
+        if v.get("type") != "page" or v.get("status") != "ok":
+            continue
+        fp = os.path.join(OUT, v.get("path", ""))
+        if not os.path.exists(fp):
+            continue
+        try:
+            html = open(fp, encoding="utf-8", errors="replace").read()
+        except Exception:
+            continue
+        for m in re.finditer(r'(?:href|src|data-link|data-href)\s*=\s*["\']([^"\']+)["\']', html, re.I):
+            h = m.group(1).strip()
+            if h.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
+                continue
+            cand = urljoin(u, h)
+            p = urlparse(cand)
+            if p.netloc.lower().split(":")[0] not in PAGE_HOSTS:
+                continue
+            if p.path.lower().endswith((".jpg", ".jpeg", ".png", ".gif", ".css", ".js",
+                                        ".ico", ".svg", ".woff", ".woff2", ".ttf", ".webp")):
+                continue
+            path = p.path
+            if path.endswith(".html"):
+                path = path[:-5]
+            if path in ("/index", "/index.html"):   # 首页别名归一
+                path = "/"
+            path, _, qs = path.partition("@@")   # 反向还原查询参数编码（改写过的链接）
+            if qs:
+                path = path + "?" + qs
+            elif p.query:                        # 未改写的原始链接：保留 query
+                path = path + "?" + p.query
+            cand = norm_url(f"{p.scheme}://{p.netloc}{path}")
+            if cand and not is_attachment(cand) and not is_junk_page(cand):
+                out.add(cand)
+    return out
+
+def norm_url(u):
+    u = u.strip()
+    if not u or u.startswith(("javascript:", "mailto:", "tel:", "data:", "sms:", "#")):
+        return None
+    u, frag = urldefrag(u)
+    p = urlparse(u)
+    if p.scheme not in ("http", "https"):
+        return None
+    host = p.netloc.lower().split(":")[0]
+    path = p.path or "/"
+    qs = []
+    if p.query:
+        for kv in p.query.split("&"):
+            k = kv.split("=", 1)[0].lower()
+            v = kv.split("=", 1)[1].lower() if "=" in kv else ""
+            if k in SKIP_QUERY_KEYS or v in SKIP_QUERY_VALS:
+                continue
+            qs.append(kv)
+    query = "&".join(qs)
+    return f"{p.scheme}://{host}{path}" + (f"?{query}" if query else "")
+
+def is_attachment(u):
+    p = urlparse(u)
+    path = unquote(p.path).lower()
+    if path.endswith(tuple(ATT_EXT)):
+        return True
+    if "/d3/" in path and p.query:   # 下载中心类动态链接
+        return True
+    return False
+
+STATIC_EXT = (".jpg", ".jpeg", ".png", ".gif", ".ico", ".svg", ".bmp", ".webp", ".tif", ".tiff")
+
+def is_static(u):
+    """<a href> 直接指向图片等静态文件时按资源处理（不追加 .html）。"""
+    return urlparse(u).path.lower().endswith(STATIC_EXT)
+
+def skip_asset(u):
+    p = urlparse(u)
+    h = p.netloc.lower().split(":")[0]
+    for bad in SKIP_ASSET_HOSTS:          # 含子域名匹配
+        if h == bad or h.endswith("." + bad):
+            return True
+    if not p.path or p.path == "/":
+        return True
+    return False
+
+def is_junk_page(u):
+    """解析噪声/静态资源伪装成的页面 URL。"""
+    p = urlparse(u)
+    path = p.path.lower()
+    if path.endswith((".css", ".js", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg",
+                      ".woff", ".woff2", ".ttf", ".webp", ".bin")):
+        return True
+    if any(s in ("http", "https") for s in [x for x in p.path.split("/") if x]):
+        return True
+    return False
+
+def sanitize_seg(seg):
+    seg = re.sub(r'[<>:"\\|?*]', "_", seg)
+    seg = seg.strip(" .")
+    if not seg:
+        seg = "_"
+    if len(seg) > 90:
+        seg = seg[:80] + "_" + hashlib.md5(seg.encode()).hexdigest()[:8]
+    return seg
+
+def local_path(u, is_page):
+    """URL → 本地相对路径。文件名使用真实 Unicode。"""
+    p = urlparse(u)
+    path = unquote(p.path)
+    if path.endswith("/"):
+        path += "index"
+    segs = [sanitize_seg(s) for s in path.split("/") if s]
+    if not segs:
+        segs = ["index"]
+    name = segs.pop()
+    if is_page:
+        name += ".html"
+    else:
+        if "." not in name or not re.search(r"\.[A-Za-z0-9]{1,6}$", name):
+            name += ".bin"
+    if p.query:
+        qs = sanitize_seg(unquote(p.query).replace("/", "_").replace("\\", "_"))[:120]
+        name = name.rsplit(".", 1)[0] + "@@" + qs + "." + name.rsplit(".", 1)[1]
+    return os.path.join(*segs, name)
+
+def fetch(u, timeout=(15, 60)):
+    for attempt in range(3):
+        try:
+            r = sess.get(u, timeout=timeout)
+            r.raise_for_status()
+            return r
+        except requests.exceptions.HTTPError as e:
+            if e.response is not None and e.response.status_code in (400, 401, 403, 404, 410):
+                raise
+            if attempt == 2:
+                raise
+            time.sleep(2 * (attempt + 1))
+        except Exception:
+            if attempt == 2:
+                raise
+            time.sleep(2 * (attempt + 1))
+
+def download_file(u, kind):
+    """下载二进制资源/附件（流式+硬时限+并发安全），返回相对路径。"""
+    if skip_asset(u):
+        return None
+    rel = local_path(u, is_page=False)
+    fp = os.path.join(OUT, rel)
+    with LOCK:
+        if u in manifest and manifest[u].get("status") == "ok" and os.path.exists(fp):
+            return rel
+        manifest[u] = {"path": rel, "type": kind, "status": "pending"}
+        save_manifest()
+    try:
+        with requests.get(u, timeout=(15, 60), stream=True,
+                          headers={"User-Agent": UA}) as r:
+            r.raise_for_status()
+            os.makedirs(os.path.dirname(fp), exist_ok=True)
+            deadline = time.time() + 600          # 单文件总时限 10 分钟，防慢速滴流卡死
+            total = 0
+            with open(fp, "wb") as f:
+                for chunk in r.iter_content(65536):
+                    if time.time() > deadline:
+                        raise TimeoutError("总时限到")
+                    f.write(chunk)
+                    total += len(chunk)
+            ct = r.headers.get("Content-Type", "")
+        # CSS: 下载其 url() 引用并改写为本地路径
+        if kind == "asset" and (rel.lower().endswith(".css") or "css" in ct):
+            try:
+                with open(fp, "rb") as f:
+                    data = f.read()
+                new_css = css_urls(data.decode("utf-8", "replace"), u)
+                with open(fp, "w", encoding="utf-8") as f:
+                    f.write(new_css)
+            except Exception:
+                pass
+        with LOCK:
+            manifest[u] = {"path": rel, "type": kind, "status": "ok",
+                           "size": total, "ct": ct}
+            save_manifest()
+        return rel
+    except Exception as e:
+        with LOCK:
+            failed.append((u, str(e)))
+            dead = isinstance(e, requests.exceptions.HTTPError) and \
+                e.response is not None and e.response.status_code in (404, 410)
+            manifest[u] = {"path": rel, "type": kind,
+                           "status": "dead" if dead else "failed", "err": str(e)[:200]}
+            save_manifest()
+            if u not in failed_logged:
+                failed_logged.add(u)
+                log(f"  [FAIL] {kind} {u} : {e}")
+        return None
+
+def css_urls(css, css_url):
+    def rep(m):
+        raw = m.group(1).strip().strip("'\"")
+        if raw.startswith(("data:", "http://", "https://", "//")):
+            absu = raw if raw.startswith(("http://", "https://")) else "https:" + raw
+        else:
+            absu = urljoin(css_url, raw)
+        if absu.startswith(("data:", "javascript:")):
+            return m.group(0)
+        rel = download_file(absu, "asset")
+        if rel:
+            from_dir = os.path.dirname(local_path(css_url, is_page=False))
+            rr = os.path.relpath(rel, from_dir).replace("\\", "/")
+            return f"url({rr})"
+        return m.group(0)
+    return re.sub(r"url\(\s*(.*?)\s*\)", rep, css, flags=re.I | re.S)
+
+def process_page(u, r):
+    soup = BeautifulSoup(r.content, "html.parser")
+    base = u
+    b = soup.find("base", href=True)
+    if b:
+        base = urljoin(u, b["href"])
+
+    rel_page = local_path(u, is_page=True)
+    page_dir = os.path.dirname(rel_page)
+
+    def to_local(absu, kind="page"):
+        """绝对 URL → 相对当前页面的本地路径。"""
+        if absu is None:
+            return None
+        rel = local_path(absu, is_page=(kind == "page"))
+        rp = os.path.relpath(rel, page_dir).replace("\\", "/")
+        return rp
+
+    new_pages, atts, assets = [], [], []
+
+    for a in soup.find_all("a", href=True):
+        h = a["href"].strip()
+        if h.startswith("#"):
+            continue
+        absu = norm_url(urljoin(base, h))
+        if absu is None:
+            continue
+        host = urlparse(absu).netloc.lower().split(":")[0]
+        if is_attachment(absu):
+            atts.append(absu)
+        elif is_static(absu):
+            assets.append(absu)                      # 图片直链按资源下载
+        elif host in PAGE_HOSTS:
+            new_pages.append(absu)
+        else:
+            ext_html_seen.add(absu)
+
+    # JS 跳转类链接：遍历全部 data-* 属性，值像 URL 的（以 / 开头、含 pid=/documentId= 等）都按链接处理
+    for tag in soup.find_all(True):
+        for attr, val in list(tag.attrs.items()):
+            if not (attr.startswith("data-") and isinstance(val, str)):
+                continue
+            v = val.strip()
+            if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(cfm|html|php|aspx)\b|pid=|documentId=", v)):
+                continue
+            absu = norm_url(urljoin(base, v))
+            if absu is None:
+                continue
+            host = urlparse(absu).netloc.lower().split(":")[0]
+            if is_attachment(absu):
+                atts.append(absu)
+            elif is_static(absu):
+                assets.append(absu)
+            elif host in PAGE_HOSTS:
+                new_pages.append(absu)
+
+    for tag in soup.find_all(["img", "script", "source", "embed", "video", "audio", "iframe"]):
+        for attr in ("src", "data", "poster"):
+            if tag.has_attr(attr):
+                absu = norm_url(urljoin(base, tag[attr].strip()))
+                if absu:
+                    assets.append(absu)
+        if tag.has_attr("srcset"):
+            for part in tag["srcset"].split(","):
+                part = part.strip()
+                if part:
+                    su = part.split()[0]
+                    absu = norm_url(urljoin(base, su))
+                    if absu:
+                        assets.append(absu)
+        if tag.name == "iframe" and tag.has_attr("src"):
+            absu = norm_url(urljoin(base, tag["src"].strip()))
+            if absu and urlparse(absu).netloc.lower().split(":")[0] in PAGE_HOSTS and not is_attachment(absu):
+                new_pages.append(absu)
+
+    for l in soup.find_all("link", href=True):
+        rels = " ".join(l.get("rel", [])).lower()
+        if any(r in rels for r in ("preconnect", "dns-prefetch", "canonical", "alternate")):
+            continue                         # 预连接/SEO 标签不是真实资源
+        absu = norm_url(urljoin(base, l["href"].strip()))
+        if absu:
+            assets.append(absu)
+
+    # ---- 改写链接为本地路径 ----
+    for a in soup.find_all("a", href=True):
+        h = a["href"].strip()
+        if h.startswith(("#", "javascript:", "mailto:", "tel:")):
+            continue
+        absu = norm_url(urljoin(base, h))
+        if absu is None:
+            continue
+        frag = ""
+        if "#" in h:
+            frag = "#" + h.split("#", 1)[1]
+        host = urlparse(absu).netloc.lower().split(":")[0]
+        if is_attachment(absu) or is_static(absu):
+            a["href"] = (to_local(absu, "asset") or absu) + frag
+        elif host in PAGE_HOSTS:
+            a["href"] = to_local(absu, "page") + frag
+
+    for tag in soup.find_all(["img", "script", "source", "embed", "video", "audio"]):
+        for attr in ("src", "data", "poster"):
+            if tag.has_attr(attr):
+                absu = norm_url(urljoin(base, tag[attr].strip()))
+                if absu:
+                    tag[attr] = to_local(absu, "asset") or absu
+        if tag.has_attr("srcset"):
+            parts = []
+            for part in tag["srcset"].split(","):
+                part = part.strip()
+                if not part:
+                    continue
+                su, *rest = part.split()
+                absu = norm_url(urljoin(base, su))
+                parts.append((to_local(absu, "asset") or absu) + (" " + " ".join(rest) if rest else ""))
+            tag["srcset"] = ", ".join(parts)
+
+    for l in soup.find_all("link", href=True):
+        rels = " ".join(l.get("rel", [])).lower()
+        if any(r in rels for r in ("preconnect", "dns-prefetch", "canonical", "alternate")):
+            continue
+        absu = norm_url(urljoin(base, l["href"].strip()))
+        if absu:
+            l["href"] = to_local(absu, "asset") or absu
+
+    for tag in soup.find_all(True):
+        for attr, val in list(tag.attrs.items()):
+            if not (attr.startswith("data-") and isinstance(val, str)):
+                continue
+            v = val.strip()
+            if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(cfm|html|php|aspx)\b|pid=|documentId=", v)):
+                continue
+            absu = norm_url(urljoin(base, v))
+            if absu:
+                tag[attr] = to_local(absu, "page") or absu
+
+    for f in soup.find_all("iframe", src=True):
+        absu = norm_url(urljoin(base, f["src"].strip()))
+        if absu:
+            host = urlparse(absu).netloc.lower().split(":")[0]
+            f["src"] = to_local(absu, "page" if host in PAGE_HOSTS else "asset") or absu
+
+    if not soup.find("meta", attrs={"charset": True}):
+        hd = soup.find("head")
+        if hd:
+            hd.insert(0, BeautifulSoup('<meta charset="utf-8">', "html.parser").find("meta"))
+
+    # 删除 <base> 标签：改写后的相对链接按页面自身目录解析，base 会劫持解析基准导致离线浏览全部 404
+    for btag in soup.find_all("base"):
+        btag.decompose()
+
+    fp = os.path.join(OUT, rel_page)
+    os.makedirs(os.path.dirname(fp), exist_ok=True)
+    with open(fp, "w", encoding="utf-8") as f:
+        f.write(str(soup))
+    manifest[u] = {"path": rel_page, "type": "page", "status": "ok", "size": len(r.content)}
+    save_manifest()
+    return new_pages, atts, assets
+
+def main():
+    # 恢复队列：上次未处理的 + 种子 + 失败页面 + 从已存页面反推的链接（断点续传）
+    q = deque()
+    for u in load_q() + [norm_url(u) for u in START_URLS] + \
+            [u for u, v in manifest.items() if v.get("type") == "page" and v.get("status") in ("failed", "pending")] + \
+            sorted(recover_queue()):
+        u = norm_url(u)
+        if u and u not in q and not is_junk_page(u):
+            q.append(u)
+    log(f"队列恢复 {len(q)} 个页面待处理")
+    pages_done = 0
+    # 先续传上次中断未完成的文件 + 重试网络错误失败的附件（死链除外）
+    pend = [(u, manifest[u].get("type", "asset")) for u, v in manifest.items()
+            if v.get("type") in ("asset", "attachment") and v.get("status") in ("pending", "failed")]
+    if pend:
+        log(f"续传未完成文件 {len(pend)} 个")
+        for u, k in pend:
+            POOL.submit(download_file, u, k)
+    while q:
+        u = q.popleft()
+        if u in visited:
+            continue
+        visited.add(u)
+        if is_junk_page(u):
+            manifest[u] = {"type": "page", "status": "dead", "err": "junk"}
+            continue
+        try:
+            r = fetch(u)
+        except Exception as e:
+            failed.append((u, str(e)))
+            # 404/410 为永久死链，标记 dead 不再重试；其它错误下次续传时重试
+            dead = isinstance(e, requests.exceptions.HTTPError) and \
+                e.response is not None and e.response.status_code in (404, 410)
+            manifest[u] = {"type": "page", "status": "dead" if dead else "failed", "err": str(e)[:200]}
+            save_manifest()
+            if not dead or u not in failed_logged:
+                failed_logged.add(u)
+                log(f"[FAIL] page {u} : {e}")
+            time.sleep(DELAY)
+            continue
+        ct = r.headers.get("Content-Type", "")
+        pth = urlparse(u).path.lower()
+        if "html" in ct.lower() or pth.endswith((".cfm", ".html", ".htm")):
+            try:
+                new_pages, atts, assets = process_page(u, r)
+                pages_done += 1
+                for a in atts:
+                    POOL.submit(download_file, a, "attachment")
+                for a in assets:
+                    POOL.submit(download_file, a, "asset")
+                for p in new_pages:
+                    if p not in visited and p not in q:
+                        q.append(p)
+                save_q(q)   # 队列持久化，中断后可续爬
+                if pages_done % 25 == 0:
+                    log(f"... 已处理 {pages_done} 页, 队列 {len(q)}, 文件总数 {len(manifest)}")
+            except Exception as e:
+                log(f"[ERR] parse {u}: {e}")
+        else:
+            download_file(u, "attachment" if is_attachment(u) else "asset")
+        if pages_done > MAX_PAGES:
+            log("达到页面数上限，停止")
+            break
+        time.sleep(DELAY)
+
+    POOL.shutdown(wait=True)   # 等待所有文件下载完成
+    log("=" * 60)
+    log(f"完成。页面 {pages_done} 个，manifest 条目 {len(manifest)} 个")
+    types = {}
+    for v in manifest.values():
+        types[v.get("type", "?")] = types.get(v.get("type", "?"), 0) + 1
+    log("类型统计:", types)
+    if failed:
+        log(f"失败 {len(failed)} 个:")
+        for u, e in failed:
+            log("  ", u, "|", e)
+    if ext_html_seen:
+        log(f"未跟进的站外 HTML 链接 {len(ext_html_seen)} 个 (示例):")
+        for u in list(ext_html_seen)[:10]:
+            log("  ", u)
+
+if __name__ == "__main__":
+    log(f"=== 开始镜像 {TARGET} @ {time.strftime('%Y-%m-%d %H:%M:%S')} ===")
+    main()