# -*- coding: utf-8 -*-
"""网站离线镜像爬虫（纯标准库版，无需 requests / beautifulsoup4）
用法:
    python mirror_stdlib.py <起始URL> [最大页数]
示例:
    python mirror_stdlib.py http://171.80.3.206:39897/
    python mirror_stdlib.py https://example.com/ 5000

行为:
    - 页面 BFS 爬取（仅目标域），统一存为 .html 便于 file:// 浏览
    - 附件（pdf/zip 等）任何域名都下载
    - CSS/JS/图片/字体等资源下载并把 HTML/CSS 内链接改写为本地相对路径
    - 断点续传（crawl_manifest.json），可重复运行补漏
    - 每请求间隔 0.3s，礼貌抓取
"""
import os, re, sys, json, time, hashlib, threading, gzip, zlib
from urllib.parse import urlparse, urljoin, urldefrag, unquote
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from collections import deque
from concurrent.futures import ThreadPoolExecutor

if len(sys.argv) < 2:
    print(__doc__)
    sys.exit(1)

TARGET = sys.argv[1].rstrip("/")
MAX_PAGES = int(sys.argv[2]) if len(sys.argv) > 2 else 3000

ROOT = os.path.dirname(os.path.abspath(__file__))
HOST = urlparse(TARGET).netloc
HOST_DIR = re.sub(r'[<>:"\\|?*]', "_", HOST)   # Windows 目录名非法字符替换（端口冒号）
OUT = os.path.join(ROOT, "mirror", HOST_DIR)
MANIFEST = os.path.join(ROOT, "crawl_manifest.json")
QUEUE_FILE = os.path.join(ROOT, "queue.json")
LOG = open(os.path.join(ROOT, "crawl_log.txt"), "a", encoding="utf-8")

PAGE_HOSTS = {HOST.lower()}
START_URLS = [TARGET]
ATT_EXT = {
    ".pdf", ".zip", ".rar", ".7z", ".stp", ".step", ".dwg", ".dxf", ".igs", ".iges",
    ".stl", ".xls", ".xlsx", ".csv", ".doc", ".docx", ".ppt", ".pptx", ".tar", ".gz",
}
SKIP_QUERY_KEYS = {"cmsfkt", "print", "lang", "language", "sprache", "session", "cfid", "cftoken", "search", "q"}
SKIP_QUERY_VALS = {"watchlist", "nl_add", "logout", "login", "cart", "basket"}
DELAY = 0.3
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
SKIP_ASSET_HOSTS = {"etracker.com", "google-analytics.com", "googletagmanager.com",
                    "doubleclick.net", "hotjar.com", "statcounter.com", "matomo.org",
                    "google.com", "facebook.com", "twitter.com", "linkedin.com"}
failed_logged = set()
LOCK = threading.RLock()
POOL = ThreadPoolExecutor(max_workers=4)

manifest = {}
if os.path.exists(MANIFEST):
    try:
        manifest = json.load(open(MANIFEST, encoding="utf-8"))
    except Exception:
        pass
visited = {u for u, v in manifest.items() if v.get("status") == "ok"}
failed = []
ext_html_seen = set()

def log(*a):
    s = " ".join(str(x) for x in a)
    print(s, flush=True)
    with LOCK:
        LOG.write(s + "\n"); LOG.flush()

def save_manifest():
    with LOCK:
        with open(MANIFEST, "w", encoding="utf-8") as f:
            json.dump(manifest, f, ensure_ascii=False, indent=1)

def load_q():
    try:
        return json.load(open(QUEUE_FILE, encoding="utf-8"))
    except Exception:
        return []

def save_q(q):
    with LOCK:
        json.dump(list(q), open(QUEUE_FILE, "w", encoding="utf-8"))

def norm_url(u):
    u = u.strip()
    if not u or u.startswith(("javascript:", "mailto:", "tel:", "data:", "sms:", "about:", "#")):
        return None
    u, frag = urldefrag(u)
    p = urlparse(u)
    if p.scheme not in ("http", "https"):
        return None
    netloc = (p.netloc or "").lower()   # 保留端口，不能 split(":") 去端口
    path = p.path or "/"
    qs = []
    if p.query:
        for kv in p.query.split("&"):
            k = kv.split("=", 1)[0].lower()
            v = kv.split("=", 1)[1].lower() if "=" in kv else ""
            if k in SKIP_QUERY_KEYS or v in SKIP_QUERY_VALS:
                continue
            qs.append(kv)
    query = "&".join(qs)
    return f"{p.scheme}://{netloc}{path}" + (f"?{query}" if query else "")

def is_attachment(u):
    p = urlparse(u)
    path = unquote(p.path).lower()
    return path.endswith(tuple(ATT_EXT))

STATIC_EXT = (".jpg", ".jpeg", ".png", ".gif", ".ico", ".svg", ".bmp", ".webp", ".tif", ".tiff")

def is_static(u):
    return urlparse(u).path.lower().endswith(STATIC_EXT)

def skip_asset(u):
    p = urlparse(u)
    h = (p.netloc or "").lower().split(":")[0]
    for bad in SKIP_ASSET_HOSTS:
        if h == bad or h.endswith("." + bad):
            return True
    if not p.path or p.path == "/":
        return True
    return False

def is_junk_page(u):
    p = urlparse(u)
    path = p.path.lower()
    return path.endswith((".css", ".js", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg",
                          ".woff", ".woff2", ".ttf", ".webp", ".bin", ".map"))

def sanitize_seg(seg):
    seg = re.sub(r'[<>:"\\|?*]', "_", seg)
    seg = seg.strip(" .")
    if not seg:
        seg = "_"
    if len(seg) > 90:
        seg = seg[:80] + "_" + hashlib.md5(seg.encode()).hexdigest()[:8]
    return seg

def local_path(u, is_page):
    p = urlparse(u)
    path = unquote(p.path)
    if path.endswith("/"):
        path += "index"
    segs = [sanitize_seg(s) for s in path.split("/") if s]
    if not segs:
        segs = ["index"]
    name = segs.pop()
    if is_page:
        name += ".html"
    else:
        if "." not in name or not re.search(r"\.[A-Za-z0-9]{1,6}$", name):
            name += ".bin"
    if p.query:
        qs = sanitize_seg(unquote(p.query).replace("/", "_").replace("\\", "_"))[:120]
        name = name.rsplit(".", 1)[0] + "@@" + qs + "." + name.rsplit(".", 1)[1]
    return os.path.join(*segs, name)

def http_get(u, timeout=30):
    """urllib GET，返回 (bytes, content_type)。处理 gzip/deflate。"""
    req = Request(u, headers={"User-Agent": UA, "Accept-Encoding": "identity"})
    resp = urlopen(req, timeout=timeout)
    data = resp.read()
    headers = resp.headers
    enc = (headers.get("Content-Encoding", "") or "").lower()
    if enc == "gzip":
        try:
            data = gzip.decompress(data)
        except Exception:
            pass
    elif enc == "deflate":
        try:
            data = zlib.decompress(data)
        except Exception:
            pass
    ct = headers.get("Content-Type", "") or ""
    return data, ct

def fetch(u, timeout=(15, 60)):
    for attempt in range(3):
        try:
            return http_get(u, timeout=timeout[1])
        except HTTPError as e:
            if e.code in (400, 401, 403, 404, 410):
                raise
            if attempt == 2:
                raise
            time.sleep(2 * (attempt + 1))
        except Exception:
            if attempt == 2:
                raise
            time.sleep(2 * (attempt + 1))

def decode(data, ct):
    """字节 -> 文本，尽力识别编码。"""
    enc = None
    m = re.search(r'charset=["\']?([\w\-]+)', ct, re.I)
    if m:
        enc = m.group(1)
    if not enc:
        head = data[:4096]
        m = re.search(br'<meta[^>]+charset=["\']?([\w\-]+)', head, re.I)
        if m:
            enc = m.group(1).decode("ascii", "ignore")
    if not enc and data.startswith(b"\xef\xbb\xbf"):
        enc = "utf-8-sig"
    if not enc:
        enc = "utf-8"
    for e in (enc, "utf-8", "gbk", "gb18030", "latin-1"):
        try:
            return data.decode(e)
        except Exception:
            continue
    return data.decode("latin-1", "replace")

def download_file(u, kind):
    """下载二进制资源/附件，返回相对路径。"""
    if skip_asset(u):
        return None
    rel = local_path(u, is_page=False)
    fp = os.path.join(OUT, rel)
    with LOCK:
        if u in manifest and manifest[u].get("status") == "ok" and os.path.exists(fp):
            return rel
        manifest[u] = {"path": rel, "type": kind, "status": "pending"}
        save_manifest()
    try:
        data, ct = http_get(u)
        os.makedirs(os.path.dirname(fp), exist_ok=True)
        with open(fp, "wb") as f:
            f.write(data)
        # CSS: 改写 url() 引用为本地路径
        if kind == "asset" and (rel.lower().endswith(".css") or "css" in ct):
            try:
                new_css = css_urls(data.decode("utf-8", "replace"), u)
                with open(fp, "w", encoding="utf-8") as f:
                    f.write(new_css)
            except Exception:
                pass
        with LOCK:
            manifest[u] = {"path": rel, "type": kind, "status": "ok",
                           "size": len(data), "ct": ct}
            save_manifest()
        return rel
    except HTTPError as e:
        with LOCK:
            failed.append((u, str(e)))
            manifest[u] = {"path": rel, "type": kind,
                           "status": "dead" if e.code in (404, 410) else "failed", "err": str(e)[:200]}
            save_manifest()
            if u not in failed_logged:
                failed_logged.add(u)
                log(f"  [FAIL] {kind} {u} : {e}")
        return None
    except Exception as e:
        with LOCK:
            failed.append((u, str(e)))
            manifest[u] = {"path": rel, "type": kind, "status": "failed", "err": str(e)[:200]}
            save_manifest()
            if u not in failed_logged:
                failed_logged.add(u)
                log(f"  [FAIL] {kind} {u} : {e}")
        return None

def css_urls(css, css_url):
    def rep(m):
        raw = m.group(1).strip().strip("'\"")
        if raw.startswith(("data:", "javascript:", "#")):
            return m.group(0)
        if raw.startswith("//"):
            absu = "http:" + raw
        else:
            absu = urljoin(css_url, raw)
        rel = download_file(absu, "asset")
        if rel:
            from_dir = os.path.dirname(local_path(css_url, is_page=False))
            rr = os.path.relpath(rel, from_dir).replace("\\", "/")
            return f"url({rr})"
        return m.group(0)
    return re.sub(r"url\(\s*(.*?)\s*\)", rep, css, flags=re.I | re.S)

def process_page(u, data, ct):
    html = decode(data, ct)
    page_rel = local_path(u, is_page=True)
    page_dir = os.path.dirname(page_rel)

    # base 标签：用于解析相对链接，随后删除
    base_for_resolve = u
    m = re.search(r'<base\b[^>]*href\s*=\s*(["\'])(.*?)\1', html, re.I | re.S)
    if m:
        base_for_resolve = urljoin(u, m.group(2).strip())
    html = re.sub(r'<base\b[^>]*/?>', '', html, flags=re.I)

    new_pages, atts, assets = [], [], []

    def to_local(absu, is_page):
        rel = local_path(absu, is_page=is_page)
        return os.path.relpath(rel, page_dir).replace("\\", "/")

    def handle_href(raw):
        v = raw.strip()
        if not v or v.startswith(("#", "javascript:", "mailto:", "tel:", "data:", "about:")):
            return raw
        frag = ""
        if "#" in v:
            v, _, fr = v.partition("#")
            frag = "#" + fr
        absu = norm_url(urljoin(base_for_resolve, v))
        if not absu:
            return raw
        host = (urlparse(absu).netloc or "").lower()
        if is_attachment(absu):
            atts.append(absu); rel = local_path(absu, False)
        elif is_static(absu):
            assets.append(absu); rel = local_path(absu, False)
        elif host in PAGE_HOSTS:
            new_pages.append(absu); rel = local_path(absu, True)
        else:
            ext_html_seen.add(absu); return raw
        return to_local(absu, is_page=not (is_attachment(absu) or is_static(absu))) + frag

    def handle_src(raw):
        v = raw.strip()
        if not v or v.startswith(("data:", "javascript:", "about:")):
            return raw
        absu = norm_url(urljoin(base_for_resolve, v))
        if not absu:
            return raw
        assets.append(absu)
        return to_local(absu, False)

    # href / src / data / poster 属性
    def sub_attr(html, attr, handler):
        pat = re.compile(r'\b(' + attr + r')\s*=\s*(["\'])(.*?)\2', re.I | re.S)
        return pat.sub(lambda m: m.group(1) + "=" + m.group(2) + handler(m.group(3)) + m.group(2), html)

    html = sub_attr(html, r'href', handle_href)
    html = sub_attr(html, r'src', handle_src)
    html = sub_attr(html, r'data', handle_src)
    html = sub_attr(html, r'poster', handle_src)

    # srcset（逗号分隔的 URL + 描述符列表）
    def sub_srcset(html):
        def rep(m):
            parts = []
            for part in m.group(3).split(","):
                part = part.strip()
                if not part:
                    continue
                toks = part.split()
                su = toks[0]
                rest = " ".join(toks[1:])
                newu = handle_src(su)
                parts.append(newu + (" " + rest if rest else ""))
            return m.group(1) + "=" + m.group(2) + ", ".join(parts) + m.group(2)
        pat = re.compile(r'\b(srcset)\s*=\s*(["\'])(.*?)\2', re.I | re.S)
        return pat.sub(rep, html)
    html = sub_srcset(html)

    # data-* 属性：仅提取 URL 用于发现页面，不改写（避免破坏 JS）
    for m in re.finditer(r'\b(data-[a-z0-9_-]+)\s*=\s*(["\'])(.*?)\2', html, re.I | re.S):
        v = m.group(3).strip()
        if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(html?|php|cfm|aspx|jsp)\b", v)):
            continue
        absu = norm_url(urljoin(base_for_resolve, v))
        if not absu:
            continue
        host = (urlparse(absu).netloc or "").lower()
        if host in PAGE_HOSTS and not is_attachment(absu) and not is_junk_page(absu):
            new_pages.append(absu)

    # 确保有 charset meta
    if not re.search(r'<meta[^>]+charset', html, re.I):
        m = re.search(r'<head\b[^>]*>', html, re.I)
        if m:
            html = html[:m.end()] + '<meta charset="utf-8">' + html[m.end():]

    fp = os.path.join(OUT, page_rel)
    os.makedirs(os.path.dirname(fp), exist_ok=True)
    with open(fp, "w", encoding="utf-8") as f:
        f.write(html)
    manifest[u] = {"path": page_rel, "type": "page", "status": "ok", "size": len(data)}
    save_manifest()
    return new_pages, atts, assets

def main():
    q = deque()
    for u in load_q() + [norm_url(u) for u in START_URLS] + \
            [u for u, v in manifest.items() if v.get("type") == "page" and v.get("status") in ("failed", "pending")]:
        u = norm_url(u)
        if u and u not in q and not is_junk_page(u):
            q.append(u)
    log(f"队列 {len(q)} 个页面待处理")
    pages_done = 0

    pend = [(u, manifest[u].get("type", "asset")) for u, v in manifest.items()
            if v.get("type") in ("asset", "attachment") and v.get("status") in ("pending", "failed")]
    if pend:
        log(f"续传未完成文件 {len(pend)} 个")
        for u, k in pend:
            POOL.submit(download_file, u, k)

    while q:
        u = q.popleft()
        if u in visited:
            continue
        visited.add(u)
        if is_junk_page(u):
            manifest[u] = {"type": "page", "status": "dead", "err": "junk"}
            continue
        try:
            data, ct = fetch(u)
        except HTTPError as e:
            failed.append((u, str(e)))
            manifest[u] = {"type": "page", "status": "dead" if e.code in (404, 410) else "failed", "err": str(e)[:200]}
            save_manifest()
            if u not in failed_logged:
                failed_logged.add(u)
                log(f"[FAIL] page {u} : {e}")
            time.sleep(DELAY)
            continue
        except Exception as e:
            failed.append((u, str(e)))
            manifest[u] = {"type": "page", "status": "failed", "err": str(e)[:200]}
            save_manifest()
            if u not in failed_logged:
                failed_logged.add(u)
                log(f"[FAIL] page {u} : {e}")
            time.sleep(DELAY)
            continue

        pth = urlparse(u).path.lower()
        if "html" in ct.lower() or pth.endswith((".htm", ".html", ".cfm", ".php", ".aspx", ".jsp")):
            try:
                new_pages, atts, assets = process_page(u, data, ct)
                pages_done += 1
                for a in atts:
                    POOL.submit(download_file, a, "attachment")
                for a in assets:
                    POOL.submit(download_file, a, "asset")
                for p in new_pages:
                    if p not in visited and p not in q:
                        q.append(p)
                save_q(q)
                if pages_done % 25 == 0:
                    log(f"... 已处理 {pages_done} 页, 队列 {len(q)}, 文件 {len(manifest)}")
            except Exception as e:
                log(f"[ERR] parse {u}: {e}")
        else:
            download_file(u, "attachment" if is_attachment(u) else "asset")

        if pages_done > MAX_PAGES:
            log("达到页面数上限，停止")
            break
        time.sleep(DELAY)

    POOL.shutdown(wait=True)
    log("=" * 60)
    log(f"完成。页面 {pages_done} 个，manifest 条目 {len(manifest)} 个")
    types = {}
    for v in manifest.values():
        types[v.get("type", "?")] = types.get(v.get("type", "?"), 0) + 1
    log("类型统计:", types)
    if failed:
        log(f"失败 {len(failed)} 个:")
        for u, e in failed:
            log("  ", u, "|", e)
    if ext_html_seen:
        log(f"未跟进的站外 HTML 链接 {len(ext_html_seen)} 个 (示例):")
        for u in list(ext_html_seen)[:10]:
            log("  ", u)

if __name__ == "__main__":
    log(f"=== 开始镜像 {TARGET} @ {time.strftime('%Y-%m-%d %H:%M:%S')} ===")
    main()