# -*- coding: utf-8 -*-
"""通用网站离线镜像爬虫（skill: website-mirror）
用法:
    python mirror.py <起始URL> [额外页面域,逗号分隔] [最大页数]
示例:
    python mirror.py https://zh.example.com/
    python mirror.py https://www.example.com/ "media.example.com,static.example.com" 5000

行为:
    - 页面 BFS 爬取（仅目标域 + 额外域），统一存为 .html 便于 file:// 浏览
    - 附件（pdf/zip/CAD 等）任何域名都下载
    - CSS/JS/图片/字体等页面资源下载并把 HTML/CSS 内链接改写为本地相对路径
    - 断点续传（crawl_manifest.json），可重复运行补漏
    - 每请求间隔 0.3s，单线程，礼貌抓取
输出:
    mirror/<域名>/...  可离线浏览的站点副本
    crawl_manifest.json 全部 URL 的下载记录
    crawl_log.txt       日志与失败清单
"""
import os, re, sys, json, time, hashlib, warnings, threading
from urllib.parse import urlparse, urljoin, urldefrag, unquote, quote
from collections import deque
from concurrent.futures import ThreadPoolExecutor
import requests
from bs4 import BeautifulSoup

warnings.filterwarnings("ignore")

if len(sys.argv) < 2:
    print(__doc__)
    sys.exit(1)

TARGET = sys.argv[1].rstrip("/")
EXTRA_HOSTS = {h.strip().lower() for h in sys.argv[2].split(",") if h.strip()} if len(sys.argv) > 2 else set()
MAX_PAGES = int(sys.argv[3]) if len(sys.argv) > 3 else 3000

ROOT = os.path.dirname(os.path.abspath(__file__))
HOST = urlparse(TARGET).netloc
HOST_DIR = re.sub(r'[<>:"\\|?*]', "_", HOST)   # Windows 目录名非法字符替换（端口冒号等）
OUT = os.path.join(ROOT, "mirror", HOST_DIR)
MANIFEST = os.path.join(ROOT, "crawl_manifest.json")
QUEUE_FILE = os.path.join(ROOT, "queue.json")
LOG = open(os.path.join(ROOT, "crawl_log.txt"), "a", encoding="utf-8")

PAGE_HOSTS = {HOST.lower()} | EXTRA_HOSTS
START_URLS = [TARGET]
ATT_EXT = {
    ".pdf", ".zip", ".rar", ".7z", ".stp", ".step", ".dwg", ".dxf", ".igs", ".iges",
    ".stl", ".easm", ".eprt", ".sldprt", ".sldasm", ".slddrw", ".xls", ".xlsx",
    ".csv", ".doc", ".docx", ".ppt", ".pptx", ".catpart", ".catproduct", ".3mf", ".jt",
    ".ifc", ".rfa", ".dgn", ".prt", ".asm", ".sdp", ".sat", ".stpz",
}
# 跳过的查询参数（表单/会话/语言切换等爬虫陷阱）
SKIP_QUERY_KEYS = {"cmsfkt", "print", "lang", "language", "sprache", "session", "cfid", "cftoken", "search", "q"}
SKIP_QUERY_VALS = {"watchlist", "nl_add", "logout", "login", "cart", "basket"}
DELAY = 0.3
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
# 统计探针/广告等注定失败的域名，直接跳过
SKIP_ASSET_HOSTS = {"etracker.com", "google-analytics.com", "googletagmanager.com",
                    "doubleclick.net", "hotjar.com", "statcounter.com", "matomo.org"}
failed_logged = set()
LOCK = threading.RLock()   # 可重入：持锁代码内还会调用 save_manifest/log
POOL = ThreadPoolExecutor(max_workers=4)   # 文件下载并发线程池

sess = requests.Session()
sess.headers.update({"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"})

manifest = {}
if os.path.exists(MANIFEST):
    try:
        manifest = json.load(open(MANIFEST, encoding="utf-8"))
    except Exception:
        pass
visited = {u for u, v in manifest.items() if v.get("status") == "ok"}
failed = []
ext_html_seen = set()

def log(*a):
    s = " ".join(str(x) for x in a)
    print(s, flush=True)
    with LOCK:
        LOG.write(s + "\n"); LOG.flush()

def save_manifest():
    with LOCK:
        with open(MANIFEST, "w", encoding="utf-8") as f:
            json.dump(manifest, f, ensure_ascii=False, indent=1)

def load_q():
    """恢复上次未处理完的页面队列。"""
    try:
        return json.load(open(QUEUE_FILE, encoding="utf-8"))
    except Exception:
        return []

def save_q(q):
    with LOCK:
        json.dump(list(q), open(QUEUE_FILE, "w", encoding="utf-8"))

def recover_queue():
    """从已保存页面的本地链接反向恢复站内页面 URL（弥补崩溃丢失的队列）。"""
    out = set()
    for u, v in manifest.items():
        if v.get("type") != "page" or v.get("status") != "ok":
            continue
        fp = os.path.join(OUT, v.get("path", ""))
        if not os.path.exists(fp):
            continue
        try:
            html = open(fp, encoding="utf-8", errors="replace").read()
        except Exception:
            continue
        for m in re.finditer(r'(?:href|src|data-link|data-href)\s*=\s*["\']([^"\']+)["\']', html, re.I):
            h = m.group(1).strip()
            if h.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
                continue
            cand = urljoin(u, h)
            p = urlparse(cand)
            if p.netloc.lower().split(":")[0] not in PAGE_HOSTS:
                continue
            if p.path.lower().endswith((".jpg", ".jpeg", ".png", ".gif", ".css", ".js",
                                        ".ico", ".svg", ".woff", ".woff2", ".ttf", ".webp")):
                continue
            path = p.path
            if path.endswith(".html"):
                path = path[:-5]
            if path in ("/index", "/index.html"):   # 首页别名归一
                path = "/"
            path, _, qs = path.partition("@@")   # 反向还原查询参数编码（改写过的链接）
            if qs:
                path = path + "?" + qs
            elif p.query:                        # 未改写的原始链接：保留 query
                path = path + "?" + p.query
            cand = norm_url(f"{p.scheme}://{p.netloc}{path}")
            if cand and not is_attachment(cand) and not is_junk_page(cand):
                out.add(cand)
    return out

def norm_url(u):
    u = u.strip()
    if not u or u.startswith(("javascript:", "mailto:", "tel:", "data:", "sms:", "#")):
        return None
    u, frag = urldefrag(u)
    p = urlparse(u)
    if p.scheme not in ("http", "https"):
        return None
    host = p.netloc.lower().split(":")[0]
    path = p.path or "/"
    qs = []
    if p.query:
        for kv in p.query.split("&"):
            k = kv.split("=", 1)[0].lower()
            v = kv.split("=", 1)[1].lower() if "=" in kv else ""
            if k in SKIP_QUERY_KEYS or v in SKIP_QUERY_VALS:
                continue
            qs.append(kv)
    query = "&".join(qs)
    return f"{p.scheme}://{host}{path}" + (f"?{query}" if query else "")

def is_attachment(u):
    p = urlparse(u)
    path = unquote(p.path).lower()
    if path.endswith(tuple(ATT_EXT)):
        return True
    if "/d3/" in path and p.query:   # 下载中心类动态链接
        return True
    return False

STATIC_EXT = (".jpg", ".jpeg", ".png", ".gif", ".ico", ".svg", ".bmp", ".webp", ".tif", ".tiff")

def is_static(u):
    """<a href> 直接指向图片等静态文件时按资源处理（不追加 .html）。"""
    return urlparse(u).path.lower().endswith(STATIC_EXT)

def skip_asset(u):
    p = urlparse(u)
    h = p.netloc.lower().split(":")[0]
    for bad in SKIP_ASSET_HOSTS:          # 含子域名匹配
        if h == bad or h.endswith("." + bad):
            return True
    if not p.path or p.path == "/":
        return True
    return False

def is_junk_page(u):
    """解析噪声/静态资源伪装成的页面 URL。"""
    p = urlparse(u)
    path = p.path.lower()
    if path.endswith((".css", ".js", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg",
                      ".woff", ".woff2", ".ttf", ".webp", ".bin")):
        return True
    if any(s in ("http", "https") for s in [x for x in p.path.split("/") if x]):
        return True
    return False

def sanitize_seg(seg):
    seg = re.sub(r'[<>:"\\|?*]', "_", seg)
    seg = seg.strip(" .")
    if not seg:
        seg = "_"
    if len(seg) > 90:
        seg = seg[:80] + "_" + hashlib.md5(seg.encode()).hexdigest()[:8]
    return seg

def local_path(u, is_page):
    """URL → 本地相对路径。文件名使用真实 Unicode。"""
    p = urlparse(u)
    path = unquote(p.path)
    if path.endswith("/"):
        path += "index"
    segs = [sanitize_seg(s) for s in path.split("/") if s]
    if not segs:
        segs = ["index"]
    name = segs.pop()
    if is_page:
        name += ".html"
    else:
        if "." not in name or not re.search(r"\.[A-Za-z0-9]{1,6}$", name):
            name += ".bin"
    if p.query:
        qs = sanitize_seg(unquote(p.query).replace("/", "_").replace("\\", "_"))[:120]
        name = name.rsplit(".", 1)[0] + "@@" + qs + "." + name.rsplit(".", 1)[1]
    return os.path.join(*segs, name)

def fetch(u, timeout=(15, 60)):
    for attempt in range(3):
        try:
            r = sess.get(u, timeout=timeout)
            r.raise_for_status()
            return r
        except requests.exceptions.HTTPError as e:
            if e.response is not None and e.response.status_code in (400, 401, 403, 404, 410):
                raise
            if attempt == 2:
                raise
            time.sleep(2 * (attempt + 1))
        except Exception:
            if attempt == 2:
                raise
            time.sleep(2 * (attempt + 1))

def download_file(u, kind):
    """下载二进制资源/附件（流式+硬时限+并发安全），返回相对路径。"""
    if skip_asset(u):
        return None
    rel = local_path(u, is_page=False)
    fp = os.path.join(OUT, rel)
    with LOCK:
        if u in manifest and manifest[u].get("status") == "ok" and os.path.exists(fp):
            return rel
        manifest[u] = {"path": rel, "type": kind, "status": "pending"}
        save_manifest()
    try:
        with requests.get(u, timeout=(15, 60), stream=True,
                          headers={"User-Agent": UA}) as r:
            r.raise_for_status()
            os.makedirs(os.path.dirname(fp), exist_ok=True)
            deadline = time.time() + 600          # 单文件总时限 10 分钟，防慢速滴流卡死
            total = 0
            with open(fp, "wb") as f:
                for chunk in r.iter_content(65536):
                    if time.time() > deadline:
                        raise TimeoutError("总时限到")
                    f.write(chunk)
                    total += len(chunk)
            ct = r.headers.get("Content-Type", "")
        # CSS: 下载其 url() 引用并改写为本地路径
        if kind == "asset" and (rel.lower().endswith(".css") or "css" in ct):
            try:
                with open(fp, "rb") as f:
                    data = f.read()
                new_css = css_urls(data.decode("utf-8", "replace"), u)
                with open(fp, "w", encoding="utf-8") as f:
                    f.write(new_css)
            except Exception:
                pass
        with LOCK:
            manifest[u] = {"path": rel, "type": kind, "status": "ok",
                           "size": total, "ct": ct}
            save_manifest()
        return rel
    except Exception as e:
        with LOCK:
            failed.append((u, str(e)))
            dead = isinstance(e, requests.exceptions.HTTPError) and \
                e.response is not None and e.response.status_code in (404, 410)
            manifest[u] = {"path": rel, "type": kind,
                           "status": "dead" if dead else "failed", "err": str(e)[:200]}
            save_manifest()
            if u not in failed_logged:
                failed_logged.add(u)
                log(f"  [FAIL] {kind} {u} : {e}")
        return None

def css_urls(css, css_url):
    def rep(m):
        raw = m.group(1).strip().strip("'\"")
        if raw.startswith(("data:", "http://", "https://", "//")):
            absu = raw if raw.startswith(("http://", "https://")) else "https:" + raw
        else:
            absu = urljoin(css_url, raw)
        if absu.startswith(("data:", "javascript:")):
            return m.group(0)
        rel = download_file(absu, "asset")
        if rel:
            from_dir = os.path.dirname(local_path(css_url, is_page=False))
            rr = os.path.relpath(rel, from_dir).replace("\\", "/")
            return f"url({rr})"
        return m.group(0)
    return re.sub(r"url\(\s*(.*?)\s*\)", rep, css, flags=re.I | re.S)

def process_page(u, r):
    soup = BeautifulSoup(r.content, "html.parser")
    base = u
    b = soup.find("base", href=True)
    if b:
        base = urljoin(u, b["href"])

    rel_page = local_path(u, is_page=True)
    page_dir = os.path.dirname(rel_page)

    def to_local(absu, kind="page"):
        """绝对 URL → 相对当前页面的本地路径。"""
        if absu is None:
            return None
        rel = local_path(absu, is_page=(kind == "page"))
        rp = os.path.relpath(rel, page_dir).replace("\\", "/")
        return rp

    new_pages, atts, assets = [], [], []

    for a in soup.find_all("a", href=True):
        h = a["href"].strip()
        if h.startswith("#"):
            continue
        absu = norm_url(urljoin(base, h))
        if absu is None:
            continue
        host = urlparse(absu).netloc.lower().split(":")[0]
        if is_attachment(absu):
            atts.append(absu)
        elif is_static(absu):
            assets.append(absu)                      # 图片直链按资源下载
        elif host in PAGE_HOSTS:
            new_pages.append(absu)
        else:
            ext_html_seen.add(absu)

    # JS 跳转类链接：遍历全部 data-* 属性，值像 URL 的（以 / 开头、含 pid=/documentId= 等）都按链接处理
    for tag in soup.find_all(True):
        for attr, val in list(tag.attrs.items()):
            if not (attr.startswith("data-") and isinstance(val, str)):
                continue
            v = val.strip()
            if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(cfm|html|php|aspx)\b|pid=|documentId=", v)):
                continue
            absu = norm_url(urljoin(base, v))
            if absu is None:
                continue
            host = urlparse(absu).netloc.lower().split(":")[0]
            if is_attachment(absu):
                atts.append(absu)
            elif is_static(absu):
                assets.append(absu)
            elif host in PAGE_HOSTS:
                new_pages.append(absu)

    for tag in soup.find_all(["img", "script", "source", "embed", "video", "audio", "iframe"]):
        for attr in ("src", "data", "poster"):
            if tag.has_attr(attr):
                absu = norm_url(urljoin(base, tag[attr].strip()))
                if absu:
                    assets.append(absu)
        if tag.has_attr("srcset"):
            for part in tag["srcset"].split(","):
                part = part.strip()
                if part:
                    su = part.split()[0]
                    absu = norm_url(urljoin(base, su))
                    if absu:
                        assets.append(absu)
        if tag.name == "iframe" and tag.has_attr("src"):
            absu = norm_url(urljoin(base, tag["src"].strip()))
            if absu and urlparse(absu).netloc.lower().split(":")[0] in PAGE_HOSTS and not is_attachment(absu):
                new_pages.append(absu)

    for l in soup.find_all("link", href=True):
        rels = " ".join(l.get("rel", [])).lower()
        if any(r in rels for r in ("preconnect", "dns-prefetch", "canonical", "alternate")):
            continue                         # 预连接/SEO 标签不是真实资源
        absu = norm_url(urljoin(base, l["href"].strip()))
        if absu:
            assets.append(absu)

    # ---- 改写链接为本地路径 ----
    for a in soup.find_all("a", href=True):
        h = a["href"].strip()
        if h.startswith(("#", "javascript:", "mailto:", "tel:")):
            continue
        absu = norm_url(urljoin(base, h))
        if absu is None:
            continue
        frag = ""
        if "#" in h:
            frag = "#" + h.split("#", 1)[1]
        host = urlparse(absu).netloc.lower().split(":")[0]
        if is_attachment(absu) or is_static(absu):
            a["href"] = (to_local(absu, "asset") or absu) + frag
        elif host in PAGE_HOSTS:
            a["href"] = to_local(absu, "page") + frag

    for tag in soup.find_all(["img", "script", "source", "embed", "video", "audio"]):
        for attr in ("src", "data", "poster"):
            if tag.has_attr(attr):
                absu = norm_url(urljoin(base, tag[attr].strip()))
                if absu:
                    tag[attr] = to_local(absu, "asset") or absu
        if tag.has_attr("srcset"):
            parts = []
            for part in tag["srcset"].split(","):
                part = part.strip()
                if not part:
                    continue
                su, *rest = part.split()
                absu = norm_url(urljoin(base, su))
                parts.append((to_local(absu, "asset") or absu) + (" " + " ".join(rest) if rest else ""))
            tag["srcset"] = ", ".join(parts)

    for l in soup.find_all("link", href=True):
        rels = " ".join(l.get("rel", [])).lower()
        if any(r in rels for r in ("preconnect", "dns-prefetch", "canonical", "alternate")):
            continue
        absu = norm_url(urljoin(base, l["href"].strip()))
        if absu:
            l["href"] = to_local(absu, "asset") or absu

    for tag in soup.find_all(True):
        for attr, val in list(tag.attrs.items()):
            if not (attr.startswith("data-") and isinstance(val, str)):
                continue
            v = val.strip()
            if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(cfm|html|php|aspx)\b|pid=|documentId=", v)):
                continue
            absu = norm_url(urljoin(base, v))
            if absu:
                tag[attr] = to_local(absu, "page") or absu

    for f in soup.find_all("iframe", src=True):
        absu = norm_url(urljoin(base, f["src"].strip()))
        if absu:
            host = urlparse(absu).netloc.lower().split(":")[0]
            f["src"] = to_local(absu, "page" if host in PAGE_HOSTS else "asset") or absu

    if not soup.find("meta", attrs={"charset": True}):
        hd = soup.find("head")
        if hd:
            hd.insert(0, BeautifulSoup('<meta charset="utf-8">', "html.parser").find("meta"))

    # 删除 <base> 标签：改写后的相对链接按页面自身目录解析，base 会劫持解析基准导致离线浏览全部 404
    for btag in soup.find_all("base"):
        btag.decompose()

    fp = os.path.join(OUT, rel_page)
    os.makedirs(os.path.dirname(fp), exist_ok=True)
    with open(fp, "w", encoding="utf-8") as f:
        f.write(str(soup))
    manifest[u] = {"path": rel_page, "type": "page", "status": "ok", "size": len(r.content)}
    save_manifest()
    return new_pages, atts, assets

def main():
    # 恢复队列：上次未处理的 + 种子 + 失败页面 + 从已存页面反推的链接（断点续传）
    q = deque()
    for u in load_q() + [norm_url(u) for u in START_URLS] + \
            [u for u, v in manifest.items() if v.get("type") == "page" and v.get("status") in ("failed", "pending")] + \
            sorted(recover_queue()):
        u = norm_url(u)
        if u and u not in q and not is_junk_page(u):
            q.append(u)
    log(f"队列恢复 {len(q)} 个页面待处理")
    pages_done = 0
    # 先续传上次中断未完成的文件 + 重试网络错误失败的附件（死链除外）
    pend = [(u, manifest[u].get("type", "asset")) for u, v in manifest.items()
            if v.get("type") in ("asset", "attachment") and v.get("status") in ("pending", "failed")]
    if pend:
        log(f"续传未完成文件 {len(pend)} 个")
        for u, k in pend:
            POOL.submit(download_file, u, k)
    while q:
        u = q.popleft()
        if u in visited:
            continue
        visited.add(u)
        if is_junk_page(u):
            manifest[u] = {"type": "page", "status": "dead", "err": "junk"}
            continue
        try:
            r = fetch(u)
        except Exception as e:
            failed.append((u, str(e)))
            # 404/410 为永久死链，标记 dead 不再重试；其它错误下次续传时重试
            dead = isinstance(e, requests.exceptions.HTTPError) and \
                e.response is not None and e.response.status_code in (404, 410)
            manifest[u] = {"type": "page", "status": "dead" if dead else "failed", "err": str(e)[:200]}
            save_manifest()
            if not dead or u not in failed_logged:
                failed_logged.add(u)
                log(f"[FAIL] page {u} : {e}")
            time.sleep(DELAY)
            continue
        ct = r.headers.get("Content-Type", "")
        pth = urlparse(u).path.lower()
        if "html" in ct.lower() or pth.endswith((".cfm", ".html", ".htm")):
            try:
                new_pages, atts, assets = process_page(u, r)
                pages_done += 1
                for a in atts:
                    POOL.submit(download_file, a, "attachment")
                for a in assets:
                    POOL.submit(download_file, a, "asset")
                for p in new_pages:
                    if p not in visited and p not in q:
                        q.append(p)
                save_q(q)   # 队列持久化，中断后可续爬
                if pages_done % 25 == 0:
                    log(f"... 已处理 {pages_done} 页, 队列 {len(q)}, 文件总数 {len(manifest)}")
            except Exception as e:
                log(f"[ERR] parse {u}: {e}")
        else:
            download_file(u, "attachment" if is_attachment(u) else "asset")
        if pages_done > MAX_PAGES:
            log("达到页面数上限，停止")
            break
        time.sleep(DELAY)

    POOL.shutdown(wait=True)   # 等待所有文件下载完成
    log("=" * 60)
    log(f"完成。页面 {pages_done} 个，manifest 条目 {len(manifest)} 个")
    types = {}
    for v in manifest.values():
        types[v.get("type", "?")] = types.get(v.get("type", "?"), 0) + 1
    log("类型统计:", types)
    if failed:
        log(f"失败 {len(failed)} 个:")
        for u, e in failed:
            log("  ", u, "|", e)
    if ext_html_seen:
        log(f"未跟进的站外 HTML 链接 {len(ext_html_seen)} 个 (示例):")
        for u in list(ext_html_seen)[:10]:
            log("  ", u)

if __name__ == "__main__":
    log(f"=== 开始镜像 {TARGET} @ {time.strftime('%Y-%m-%d %H:%M:%S')} ===")
    main()