wangruiguo/OnebotCatalog
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

首次提交: OnebotCatalog 项目代码与文档（含 NX 按需生成服务二期、后台、一键启动）

Browse Source
This commit is contained in:
wangruiguo
2026-09-03 17:55:45 +08:00
commit fafa86d3a6
241 changed files with 78656 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
blog_mirror/crawl_log.txt Normal file
View File

@@ -0,0 +1,21 @@
=== 开始镜像 http://171.80.3.206:39897 @ 2026-08-27 21:46:18 ===
队列 1 个页面待处理
[FAIL] page http://171.80.3.206/ : <urlopen error [WinError 10061] 由于目标计算机积极拒绝，无法连接。>
============================================================
完成。页面 0 个，manifest 条目 1 个
类型统计: {'page': 1}
失败 1 个:
http://171.80.3.206/ | <urlopen error [WinError 10061] 由于目标计算机积极拒绝，无法连接。>
=== 开始镜像 http://171.80.3.206:39897 @ 2026-08-27 23:07:26 ===
队列 1 个页面待处理
[FAIL] page http://171.80.3.206/ : <urlopen error [WinError 10061] 由于目标计算机积极拒绝，无法连接。>
============================================================
完成。页面 0 个，manifest 条目 1 个
类型统计: {'page': 1}
失败 1 个:
http://171.80.3.206/ | <urlopen error [WinError 10061] 由于目标计算机积极拒绝，无法连接。>
=== 开始镜像 http://171.80.3.206:39897 @ 2026-08-27 23:12:02 ===
队列 1 个页面待处理
============================================================
完成。页面 1 个，manifest 条目 8 个
类型统计: {'page': 6, 'asset': 2}

47
blog_mirror/crawl_manifest.json Normal file
View File

@@ -0,0 +1,47 @@
{
"http://171.80.3.206:39897/": {
"path": "index.html",
"type": "page",
"status": "ok",
"size": 5016
},
"http://171.80.3.206:39897/logo.svg": {
"path": "logo.svg",
"type": "asset",
"status": "ok",
"size": 2042,
"ct": "image/svg+xml"
},
"http://171.80.3.206:39897/assets/index-DTlSLaLS.js": {
"path": "assets\\index-DTlSLaLS.js",
"type": "asset",
"status": "ok",
"size": 183464,
"ct": "text/javascript; charset=utf-8"
},
"http://171.80.3.206:39897/assets/vendor-vue-DiG09V7Q.js": {
"type": "page",
"status": "dead",
"err": "junk"
},
"http://171.80.3.206:39897/assets/vendor-i18n-F92pr2wT.js": {
"type": "page",
"status": "dead",
"err": "junk"
},
"http://171.80.3.206:39897/assets/vendor-misc-CbSuWkr5.js": {
"type": "page",
"status": "dead",
"err": "junk"
},
"http://171.80.3.206:39897/assets/vendor-misc-DB0Q8XAf.css": {
"type": "page",
"status": "dead",
"err": "junk"
},
"http://171.80.3.206:39897/assets/index-Dk4t2Wci.css": {
"type": "page",
"status": "dead",
"err": "junk"
}
}

554
blog_mirror/mirror.py Normal file
View File

@@ -0,0 +1,554 @@
# -*- coding: utf-8 -*-
"""通用网站离线镜像爬虫（skill: website-mirror）
用法:
python mirror.py <起始URL> [额外页面域,逗号分隔] [最大页数]
示例:
python mirror.py https://zh.example.com/
python mirror.py https://www.example.com/ "media.example.com,static.example.com" 5000
行为:
- 页面 BFS 爬取（仅目标域 + 额外域），统一存为 .html 便于 file:// 浏览
- 附件（pdf/zip/CAD 等）任何域名都下载
- CSS/JS/图片/字体等页面资源下载并把 HTML/CSS 内链接改写为本地相对路径
- 断点续传（crawl_manifest.json），可重复运行补漏
- 每请求间隔 0.3s，单线程，礼貌抓取
输出:
mirror/<域名>/... 可离线浏览的站点副本
crawl_manifest.json 全部 URL 的下载记录
crawl_log.txt 日志与失败清单
"""
import os, re, sys, json, time, hashlib, warnings, threading
from urllib.parse import urlparse, urljoin, urldefrag, unquote, quote
from collections import deque
from concurrent.futures import ThreadPoolExecutor
import requests
from bs4 import BeautifulSoup
warnings.filterwarnings("ignore")
if len(sys.argv) < 2:
print(__doc__)
sys.exit(1)
TARGET = sys.argv[1].rstrip("/")
EXTRA_HOSTS = {h.strip().lower() for h in sys.argv[2].split(",") if h.strip()} if len(sys.argv) > 2 else set()
MAX_PAGES = int(sys.argv[3]) if len(sys.argv) > 3 else 3000
ROOT = os.path.dirname(os.path.abspath(__file__))
HOST = urlparse(TARGET).netloc
HOST_DIR = re.sub(r'[<>:"\\|?*]', "_", HOST) # Windows 目录名非法字符替换（端口冒号等）
OUT = os.path.join(ROOT, "mirror", HOST_DIR)
MANIFEST = os.path.join(ROOT, "crawl_manifest.json")
QUEUE_FILE = os.path.join(ROOT, "queue.json")
LOG = open(os.path.join(ROOT, "crawl_log.txt"), "a", encoding="utf-8")
PAGE_HOSTS = {HOST.lower()} | EXTRA_HOSTS
START_URLS = [TARGET]
ATT_EXT = {
".pdf", ".zip", ".rar", ".7z", ".stp", ".step", ".dwg", ".dxf", ".igs", ".iges",
".stl", ".easm", ".eprt", ".sldprt", ".sldasm", ".slddrw", ".xls", ".xlsx",
".csv", ".doc", ".docx", ".ppt", ".pptx", ".catpart", ".catproduct", ".3mf", ".jt",
".ifc", ".rfa", ".dgn", ".prt", ".asm", ".sdp", ".sat", ".stpz",
}
# 跳过的查询参数（表单/会话/语言切换等爬虫陷阱）
SKIP_QUERY_KEYS = {"cmsfkt", "print", "lang", "language", "sprache", "session", "cfid", "cftoken", "search", "q"}
SKIP_QUERY_VALS = {"watchlist", "nl_add", "logout", "login", "cart", "basket"}
DELAY = 0.3
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
# 统计探针/广告等注定失败的域名，直接跳过
SKIP_ASSET_HOSTS = {"etracker.com", "google-analytics.com", "googletagmanager.com",
"doubleclick.net", "hotjar.com", "statcounter.com", "matomo.org"}
failed_logged = set()
LOCK = threading.RLock() # 可重入：持锁代码内还会调用 save_manifest/log
POOL = ThreadPoolExecutor(max_workers=4) # 文件下载并发线程池
sess = requests.Session()
sess.headers.update({"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"})
manifest = {}
if os.path.exists(MANIFEST):
try:
manifest = json.load(open(MANIFEST, encoding="utf-8"))
except Exception:
pass
visited = {u for u, v in manifest.items() if v.get("status") == "ok"}
failed = []
ext_html_seen = set()
def log(*a):
s = " ".join(str(x) for x in a)
print(s, flush=True)
with LOCK:
LOG.write(s + "\n"); LOG.flush()
def save_manifest():
with LOCK:
with open(MANIFEST, "w", encoding="utf-8") as f:
json.dump(manifest, f, ensure_ascii=False, indent=1)
def load_q():
"""恢复上次未处理完的页面队列。"""
try:
return json.load(open(QUEUE_FILE, encoding="utf-8"))
except Exception:
return []
def save_q(q):
with LOCK:
json.dump(list(q), open(QUEUE_FILE, "w", encoding="utf-8"))
def recover_queue():
"""从已保存页面的本地链接反向恢复站内页面 URL（弥补崩溃丢失的队列）。"""
out = set()
for u, v in manifest.items():
if v.get("type") != "page" or v.get("status") != "ok":
continue
fp = os.path.join(OUT, v.get("path", ""))
if not os.path.exists(fp):
continue
try:
html = open(fp, encoding="utf-8", errors="replace").read()
except Exception:
continue
for m in re.finditer(r'(?:href|src|data-link|data-href)\s*=\s*["\']([^"\']+)["\']', html, re.I):
h = m.group(1).strip()
if h.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
continue
cand = urljoin(u, h)
p = urlparse(cand)
if p.netloc.lower().split(":")[0] not in PAGE_HOSTS:
continue
if p.path.lower().endswith((".jpg", ".jpeg", ".png", ".gif", ".css", ".js",
".ico", ".svg", ".woff", ".woff2", ".ttf", ".webp")):
continue
path = p.path
if path.endswith(".html"):
path = path[:-5]
if path in ("/index", "/index.html"): # 首页别名归一
path = "/"
path, _, qs = path.partition("@@") # 反向还原查询参数编码（改写过的链接）
if qs:
path = path + "?" + qs
elif p.query: # 未改写的原始链接：保留 query
path = path + "?" + p.query
cand = norm_url(f"{p.scheme}://{p.netloc}{path}")
if cand and not is_attachment(cand) and not is_junk_page(cand):
out.add(cand)
return out
def norm_url(u):
u = u.strip()
if not u or u.startswith(("javascript:", "mailto:", "tel:", "data:", "sms:", "#")):
return None
u, frag = urldefrag(u)
p = urlparse(u)
if p.scheme not in ("http", "https"):
return None
host = p.netloc.lower().split(":")[0]
path = p.path or "/"
qs = []
if p.query:
for kv in p.query.split("&"):
k = kv.split("=", 1)[0].lower()
v = kv.split("=", 1)[1].lower() if "=" in kv else ""
if k in SKIP_QUERY_KEYS or v in SKIP_QUERY_VALS:
continue
qs.append(kv)
query = "&".join(qs)
return f"{p.scheme}://{host}{path}" + (f"?{query}" if query else "")
def is_attachment(u):
p = urlparse(u)
path = unquote(p.path).lower()
if path.endswith(tuple(ATT_EXT)):
return True
if "/d3/" in path and p.query: # 下载中心类动态链接
return True
return False
STATIC_EXT = (".jpg", ".jpeg", ".png", ".gif", ".ico", ".svg", ".bmp", ".webp", ".tif", ".tiff")
def is_static(u):
"""<a href> 直接指向图片等静态文件时按资源处理（不追加 .html）。"""
return urlparse(u).path.lower().endswith(STATIC_EXT)
def skip_asset(u):
p = urlparse(u)
h = p.netloc.lower().split(":")[0]
for bad in SKIP_ASSET_HOSTS: # 含子域名匹配
if h == bad or h.endswith("." + bad):
return True
if not p.path or p.path == "/":
return True
return False
def is_junk_page(u):
"""解析噪声/静态资源伪装成的页面 URL。"""
p = urlparse(u)
path = p.path.lower()
if path.endswith((".css", ".js", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg",
".woff", ".woff2", ".ttf", ".webp", ".bin")):
return True
if any(s in ("http", "https") for s in [x for x in p.path.split("/") if x]):
return True
return False
def sanitize_seg(seg):
seg = re.sub(r'[<>:"\\|?*]', "_", seg)
seg = seg.strip(" .")
if not seg:
seg = "_"
if len(seg) > 90:
seg = seg[:80] + "_" + hashlib.md5(seg.encode()).hexdigest()[:8]
return seg
def local_path(u, is_page):
"""URL → 本地相对路径。文件名使用真实 Unicode。"""
p = urlparse(u)
path = unquote(p.path)
if path.endswith("/"):
path += "index"
segs = [sanitize_seg(s) for s in path.split("/") if s]
if not segs:
segs = ["index"]
name = segs.pop()
if is_page:
name += ".html"
else:
if "." not in name or not re.search(r"\.[A-Za-z0-9]{1,6}$", name):
name += ".bin"
if p.query:
qs = sanitize_seg(unquote(p.query).replace("/", "_").replace("\\", "_"))[:120]
name = name.rsplit(".", 1)[0] + "@@" + qs + "." + name.rsplit(".", 1)[1]
return os.path.join(*segs, name)
def fetch(u, timeout=(15, 60)):
for attempt in range(3):
try:
r = sess.get(u, timeout=timeout)
r.raise_for_status()
return r
except requests.exceptions.HTTPError as e:
if e.response is not None and e.response.status_code in (400, 401, 403, 404, 410):
raise
if attempt == 2:
raise
time.sleep(2 * (attempt + 1))
except Exception:
if attempt == 2:
raise
time.sleep(2 * (attempt + 1))
def download_file(u, kind):
"""下载二进制资源/附件（流式+硬时限+并发安全），返回相对路径。"""
if skip_asset(u):
return None
rel = local_path(u, is_page=False)
fp = os.path.join(OUT, rel)
with LOCK:
if u in manifest and manifest[u].get("status") == "ok" and os.path.exists(fp):
return rel
manifest[u] = {"path": rel, "type": kind, "status": "pending"}
save_manifest()
try:
with requests.get(u, timeout=(15, 60), stream=True,
headers={"User-Agent": UA}) as r:
r.raise_for_status()
os.makedirs(os.path.dirname(fp), exist_ok=True)
deadline = time.time() + 600 # 单文件总时限 10 分钟，防慢速滴流卡死
total = 0
with open(fp, "wb") as f:
for chunk in r.iter_content(65536):
if time.time() > deadline:
raise TimeoutError("总时限到")
f.write(chunk)
total += len(chunk)
ct = r.headers.get("Content-Type", "")
# CSS: 下载其 url() 引用并改写为本地路径
if kind == "asset" and (rel.lower().endswith(".css") or "css" in ct):
try:
with open(fp, "rb") as f:
data = f.read()
new_css = css_urls(data.decode("utf-8", "replace"), u)
with open(fp, "w", encoding="utf-8") as f:
f.write(new_css)
except Exception:
pass
with LOCK:
manifest[u] = {"path": rel, "type": kind, "status": "ok",
"size": total, "ct": ct}
save_manifest()
return rel
except Exception as e:
with LOCK:
failed.append((u, str(e)))
dead = isinstance(e, requests.exceptions.HTTPError) and \
e.response is not None and e.response.status_code in (404, 410)
manifest[u] = {"path": rel, "type": kind,
"status": "dead" if dead else "failed", "err": str(e)[:200]}
save_manifest()
if u not in failed_logged:
failed_logged.add(u)
log(f" [FAIL] {kind} {u} : {e}")
return None
def css_urls(css, css_url):
def rep(m):
raw = m.group(1).strip().strip("'\"")
if raw.startswith(("data:", "http://", "https://", "//")):
absu = raw if raw.startswith(("http://", "https://")) else "https:" + raw
else:
absu = urljoin(css_url, raw)
if absu.startswith(("data:", "javascript:")):
return m.group(0)
rel = download_file(absu, "asset")
if rel:
from_dir = os.path.dirname(local_path(css_url, is_page=False))
rr = os.path.relpath(rel, from_dir).replace("\\", "/")
return f"url({rr})"
return m.group(0)
return re.sub(r"url\(\s*(.*?)\s*\)", rep, css, flags=re.I | re.S)
def process_page(u, r):
soup = BeautifulSoup(r.content, "html.parser")
base = u
b = soup.find("base", href=True)
if b:
base = urljoin(u, b["href"])
rel_page = local_path(u, is_page=True)
page_dir = os.path.dirname(rel_page)
def to_local(absu, kind="page"):
"""绝对 URL → 相对当前页面的本地路径。"""
if absu is None:
return None
rel = local_path(absu, is_page=(kind == "page"))
rp = os.path.relpath(rel, page_dir).replace("\\", "/")
return rp
new_pages, atts, assets = [], [], []
for a in soup.find_all("a", href=True):
h = a["href"].strip()
if h.startswith("#"):
continue
absu = norm_url(urljoin(base, h))
if absu is None:
continue
host = urlparse(absu).netloc.lower().split(":")[0]
if is_attachment(absu):
atts.append(absu)
elif is_static(absu):
assets.append(absu) # 图片直链按资源下载
elif host in PAGE_HOSTS:
new_pages.append(absu)
else:
ext_html_seen.add(absu)
# JS 跳转类链接：遍历全部 data-* 属性，值像 URL 的（以 / 开头、含 pid=/documentId= 等）都按链接处理
for tag in soup.find_all(True):
for attr, val in list(tag.attrs.items()):
if not (attr.startswith("data-") and isinstance(val, str)):
continue
v = val.strip()
if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(cfm|html|php|aspx)\b|pid=|documentId=", v)):
continue
absu = norm_url(urljoin(base, v))
if absu is None:
continue
host = urlparse(absu).netloc.lower().split(":")[0]
if is_attachment(absu):
atts.append(absu)
elif is_static(absu):
assets.append(absu)
elif host in PAGE_HOSTS:
new_pages.append(absu)
for tag in soup.find_all(["img", "script", "source", "embed", "video", "audio", "iframe"]):
for attr in ("src", "data", "poster"):
if tag.has_attr(attr):
absu = norm_url(urljoin(base, tag[attr].strip()))
if absu:
assets.append(absu)
if tag.has_attr("srcset"):
for part in tag["srcset"].split(","):
part = part.strip()
if part:
su = part.split()[0]
absu = norm_url(urljoin(base, su))
if absu:
assets.append(absu)
if tag.name == "iframe" and tag.has_attr("src"):
absu = norm_url(urljoin(base, tag["src"].strip()))
if absu and urlparse(absu).netloc.lower().split(":")[0] in PAGE_HOSTS and not is_attachment(absu):
new_pages.append(absu)
for l in soup.find_all("link", href=True):
rels = " ".join(l.get("rel", [])).lower()
if any(r in rels for r in ("preconnect", "dns-prefetch", "canonical", "alternate")):
continue # 预连接/SEO 标签不是真实资源
absu = norm_url(urljoin(base, l["href"].strip()))
if absu:
assets.append(absu)
# ---- 改写链接为本地路径 ----
for a in soup.find_all("a", href=True):
h = a["href"].strip()
if h.startswith(("#", "javascript:", "mailto:", "tel:")):
continue
absu = norm_url(urljoin(base, h))
if absu is None:
continue
frag = ""
if "#" in h:
frag = "#" + h.split("#", 1)[1]
host = urlparse(absu).netloc.lower().split(":")[0]
if is_attachment(absu) or is_static(absu):
a["href"] = (to_local(absu, "asset") or absu) + frag
elif host in PAGE_HOSTS:
a["href"] = to_local(absu, "page") + frag
for tag in soup.find_all(["img", "script", "source", "embed", "video", "audio"]):
for attr in ("src", "data", "poster"):
if tag.has_attr(attr):
absu = norm_url(urljoin(base, tag[attr].strip()))
if absu:
tag[attr] = to_local(absu, "asset") or absu
if tag.has_attr("srcset"):
parts = []
for part in tag["srcset"].split(","):
part = part.strip()
if not part:
continue
su, *rest = part.split()
absu = norm_url(urljoin(base, su))
parts.append((to_local(absu, "asset") or absu) + (" " + " ".join(rest) if rest else ""))
tag["srcset"] = ", ".join(parts)
for l in soup.find_all("link", href=True):
rels = " ".join(l.get("rel", [])).lower()
if any(r in rels for r in ("preconnect", "dns-prefetch", "canonical", "alternate")):
continue
absu = norm_url(urljoin(base, l["href"].strip()))
if absu:
l["href"] = to_local(absu, "asset") or absu
for tag in soup.find_all(True):
for attr, val in list(tag.attrs.items()):
if not (attr.startswith("data-") and isinstance(val, str)):
continue
v = val.strip()
if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(cfm|html|php|aspx)\b|pid=|documentId=", v)):
continue
absu = norm_url(urljoin(base, v))
if absu:
tag[attr] = to_local(absu, "page") or absu
for f in soup.find_all("iframe", src=True):
absu = norm_url(urljoin(base, f["src"].strip()))
if absu:
host = urlparse(absu).netloc.lower().split(":")[0]
f["src"] = to_local(absu, "page" if host in PAGE_HOSTS else "asset") or absu
if not soup.find("meta", attrs={"charset": True}):
hd = soup.find("head")
if hd:
hd.insert(0, BeautifulSoup('<meta charset="utf-8">', "html.parser").find("meta"))
# 删除 <base> 标签：改写后的相对链接按页面自身目录解析，base 会劫持解析基准导致离线浏览全部 404
for btag in soup.find_all("base"):
btag.decompose()
fp = os.path.join(OUT, rel_page)
os.makedirs(os.path.dirname(fp), exist_ok=True)
with open(fp, "w", encoding="utf-8") as f:
f.write(str(soup))
manifest[u] = {"path": rel_page, "type": "page", "status": "ok", "size": len(r.content)}
save_manifest()
return new_pages, atts, assets
def main():
# 恢复队列：上次未处理的 + 种子 + 失败页面 + 从已存页面反推的链接（断点续传）
q = deque()
for u in load_q() + [norm_url(u) for u in START_URLS] + \
[u for u, v in manifest.items() if v.get("type") == "page" and v.get("status") in ("failed", "pending")] + \
sorted(recover_queue()):
u = norm_url(u)
if u and u not in q and not is_junk_page(u):
q.append(u)
log(f"队列恢复 {len(q)} 个页面待处理")
pages_done = 0
# 先续传上次中断未完成的文件 + 重试网络错误失败的附件（死链除外）
pend = [(u, manifest[u].get("type", "asset")) for u, v in manifest.items()
if v.get("type") in ("asset", "attachment") and v.get("status") in ("pending", "failed")]
if pend:
log(f"续传未完成文件 {len(pend)} 个")
for u, k in pend:
POOL.submit(download_file, u, k)
while q:
u = q.popleft()
if u in visited:
continue
visited.add(u)
if is_junk_page(u):
manifest[u] = {"type": "page", "status": "dead", "err": "junk"}
continue
try:
r = fetch(u)
except Exception as e:
failed.append((u, str(e)))
# 404/410 为永久死链，标记 dead 不再重试；其它错误下次续传时重试
dead = isinstance(e, requests.exceptions.HTTPError) and \
e.response is not None and e.response.status_code in (404, 410)
manifest[u] = {"type": "page", "status": "dead" if dead else "failed", "err": str(e)[:200]}
save_manifest()
if not dead or u not in failed_logged:
failed_logged.add(u)
log(f"[FAIL] page {u} : {e}")
time.sleep(DELAY)
continue
ct = r.headers.get("Content-Type", "")
pth = urlparse(u).path.lower()
if "html" in ct.lower() or pth.endswith((".cfm", ".html", ".htm")):
try:
new_pages, atts, assets = process_page(u, r)
pages_done += 1
for a in atts:
POOL.submit(download_file, a, "attachment")
for a in assets:
POOL.submit(download_file, a, "asset")
for p in new_pages:
if p not in visited and p not in q:
q.append(p)
save_q(q) # 队列持久化，中断后可续爬
if pages_done % 25 == 0:
log(f"... 已处理 {pages_done} 页, 队列 {len(q)}, 文件总数 {len(manifest)}")
except Exception as e:
log(f"[ERR] parse {u}: {e}")
else:
download_file(u, "attachment" if is_attachment(u) else "asset")
if pages_done > MAX_PAGES:
log("达到页面数上限，停止")
break
time.sleep(DELAY)
POOL.shutdown(wait=True) # 等待所有文件下载完成
log("=" * 60)
log(f"完成。页面 {pages_done} 个，manifest 条目 {len(manifest)} 个")
types = {}
for v in manifest.values():
types[v.get("type", "?")] = types.get(v.get("type", "?"), 0) + 1
log("类型统计:", types)
if failed:
log(f"失败 {len(failed)} 个:")
for u, e in failed:
log(" ", u, "|", e)
if ext_html_seen:
log(f"未跟进的站外 HTML 链接 {len(ext_html_seen)} 个 (示例):")
for u in list(ext_html_seen)[:10]:
log(" ", u)
if __name__ == "__main__":
log(f"=== 开始镜像 {TARGET} @ {time.strftime('%Y-%m-%d %H:%M:%S')} ===")
main()

100
blog_mirror/mirror/171.80.3.206_39897/assets/index-DTlSLaLS.js Normal file
View File

File diff suppressed because one or more lines are too long

18
blog_mirror/mirror/171.80.3.206_39897/index.html Normal file
View File

@@ -0,0 +1,18 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="logo.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Forestapi - AI API Gateway</title>
<script type="module" crossorigin src="assets/index-DTlSLaLS.js"></script>
<link rel="modulepreload" crossorigin href="assets/vendor-vue-DiG09V7Q.js.html">
<link rel="modulepreload" crossorigin href="assets/vendor-i18n-F92pr2wT.js.html">
<link rel="modulepreload" crossorigin href="assets/vendor-misc-CbSuWkr5.js.html">

32
blog_mirror/mirror/171.80.3.206_39897/logo.svg Normal file
View File

@@ -0,0 +1,32 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" role="img" aria-labelledby="title desc">
<title id="title">Sub2API</title>
<desc id="desc">An interlocking S symbol representing subscription routing into APIs.</desc>
<defs>
<linearGradient id="s2a-bg" x1="72" y1="44" x2="442" y2="478" gradientUnits="userSpaceOnUse">
<stop stop-color="#142B56"/>
<stop offset=".52" stop-color="#0A1A39"/>
<stop offset="1" stop-color="#061127"/>
</linearGradient>
<radialGradient id="s2a-ambient" cx="0" cy="0" r="1" gradientUnits="userSpaceOnUse" gradientTransform="translate(168 92) rotate(51) scale(342 382)">
<stop stop-color="#3E68B0" stop-opacity=".28"/>
<stop offset="1" stop-color="#3E68B0" stop-opacity="0"/>
</radialGradient>
<linearGradient id="s2a-brand" x1="4" y1="4" x2="20" y2="20" gradientUnits="userSpaceOnUse">
<stop stop-color="#79F4BD"/>
<stop offset=".48" stop-color="#39D9E7"/>
<stop offset="1" stop-color="#3875F6"/>
</linearGradient>
<filter id="s2a-shadow" x="-25%" y="-25%" width="150%" height="165%" color-interpolation-filters="sRGB">
<feDropShadow dx="0" dy=".55" stdDeviation=".65" flood-color="#020817" flood-opacity=".34"/>
</filter>
</defs>
<rect x="16" y="16" width="480" height="480" rx="120" fill="url(#s2a-bg)"/>
<rect x="16" y="16" width="480" height="480" rx="120" fill="url(#s2a-ambient)"/>
<rect x="16.75" y="16.75" width="478.5" height="478.5" rx="119.25" fill="none" stroke="#A5BFFF" stroke-opacity=".16" stroke-width="1.5"/>
<g transform="translate(52 52) scale(17)" fill="none" stroke="url(#s2a-brand)" stroke-linecap="round" stroke-linejoin="round" stroke-width="2.7" filter="url(#s2a-shadow)">
<path d="m19.25 7.65-2.55-3.2a1.33 1.33 0 0 0-1.03-.5H8.58c-.34 0-.67.13-.91.37L4.15 7.65c-.93.88-.6 1.52.65 2.3l9.55 5.97"/>
<path d="m4.75 16.35 2.55 3.2c.25.31.63.5 1.03.5h7.09c.34 0 .67-.13.91-.37l3.52-3.33c.93-.88.6-1.52-.65-2.3L9.65 8.08"/>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.0 KiB

477
blog_mirror/mirror_stdlib.py Normal file
View File

@@ -0,0 +1,477 @@
# -*- coding: utf-8 -*-
"""网站离线镜像爬虫（纯标准库版，无需 requests / beautifulsoup4）
用法:
python mirror_stdlib.py <起始URL> [最大页数]
示例:
python mirror_stdlib.py http://171.80.3.206:39897/
python mirror_stdlib.py https://example.com/ 5000
行为:
- 页面 BFS 爬取（仅目标域），统一存为 .html 便于 file:// 浏览
- 附件（pdf/zip 等）任何域名都下载
- CSS/JS/图片/字体等资源下载并把 HTML/CSS 内链接改写为本地相对路径
- 断点续传（crawl_manifest.json），可重复运行补漏
- 每请求间隔 0.3s，礼貌抓取
"""
import os, re, sys, json, time, hashlib, threading, gzip, zlib
from urllib.parse import urlparse, urljoin, urldefrag, unquote
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from collections import deque
from concurrent.futures import ThreadPoolExecutor
if len(sys.argv) < 2:
print(__doc__)
sys.exit(1)
TARGET = sys.argv[1].rstrip("/")
MAX_PAGES = int(sys.argv[2]) if len(sys.argv) > 2 else 3000
ROOT = os.path.dirname(os.path.abspath(__file__))
HOST = urlparse(TARGET).netloc
HOST_DIR = re.sub(r'[<>:"\\|?*]', "_", HOST) # Windows 目录名非法字符替换（端口冒号）
OUT = os.path.join(ROOT, "mirror", HOST_DIR)
MANIFEST = os.path.join(ROOT, "crawl_manifest.json")
QUEUE_FILE = os.path.join(ROOT, "queue.json")
LOG = open(os.path.join(ROOT, "crawl_log.txt"), "a", encoding="utf-8")
PAGE_HOSTS = {HOST.lower()}
START_URLS = [TARGET]
ATT_EXT = {
".pdf", ".zip", ".rar", ".7z", ".stp", ".step", ".dwg", ".dxf", ".igs", ".iges",
".stl", ".xls", ".xlsx", ".csv", ".doc", ".docx", ".ppt", ".pptx", ".tar", ".gz",
}
SKIP_QUERY_KEYS = {"cmsfkt", "print", "lang", "language", "sprache", "session", "cfid", "cftoken", "search", "q"}
SKIP_QUERY_VALS = {"watchlist", "nl_add", "logout", "login", "cart", "basket"}
DELAY = 0.3
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
SKIP_ASSET_HOSTS = {"etracker.com", "google-analytics.com", "googletagmanager.com",
"doubleclick.net", "hotjar.com", "statcounter.com", "matomo.org",
"google.com", "facebook.com", "twitter.com", "linkedin.com"}
failed_logged = set()
LOCK = threading.RLock()
POOL = ThreadPoolExecutor(max_workers=4)
manifest = {}
if os.path.exists(MANIFEST):
try:
manifest = json.load(open(MANIFEST, encoding="utf-8"))
except Exception:
pass
visited = {u for u, v in manifest.items() if v.get("status") == "ok"}
failed = []
ext_html_seen = set()
def log(*a):
s = " ".join(str(x) for x in a)
print(s, flush=True)
with LOCK:
LOG.write(s + "\n"); LOG.flush()
def save_manifest():
with LOCK:
with open(MANIFEST, "w", encoding="utf-8") as f:
json.dump(manifest, f, ensure_ascii=False, indent=1)
def load_q():
try:
return json.load(open(QUEUE_FILE, encoding="utf-8"))
except Exception:
return []
def save_q(q):
with LOCK:
json.dump(list(q), open(QUEUE_FILE, "w", encoding="utf-8"))
def norm_url(u):
u = u.strip()
if not u or u.startswith(("javascript:", "mailto:", "tel:", "data:", "sms:", "about:", "#")):
return None
u, frag = urldefrag(u)
p = urlparse(u)
if p.scheme not in ("http", "https"):
return None
netloc = (p.netloc or "").lower() # 保留端口，不能 split(":") 去端口
path = p.path or "/"
qs = []
if p.query:
for kv in p.query.split("&"):
k = kv.split("=", 1)[0].lower()
v = kv.split("=", 1)[1].lower() if "=" in kv else ""
if k in SKIP_QUERY_KEYS or v in SKIP_QUERY_VALS:
continue
qs.append(kv)
query = "&".join(qs)
return f"{p.scheme}://{netloc}{path}" + (f"?{query}" if query else "")
def is_attachment(u):
p = urlparse(u)
path = unquote(p.path).lower()
return path.endswith(tuple(ATT_EXT))
STATIC_EXT = (".jpg", ".jpeg", ".png", ".gif", ".ico", ".svg", ".bmp", ".webp", ".tif", ".tiff")
def is_static(u):
return urlparse(u).path.lower().endswith(STATIC_EXT)
def skip_asset(u):
p = urlparse(u)
h = (p.netloc or "").lower().split(":")[0]
for bad in SKIP_ASSET_HOSTS:
if h == bad or h.endswith("." + bad):
return True
if not p.path or p.path == "/":
return True
return False
def is_junk_page(u):
p = urlparse(u)
path = p.path.lower()
return path.endswith((".css", ".js", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg",
".woff", ".woff2", ".ttf", ".webp", ".bin", ".map"))
def sanitize_seg(seg):
seg = re.sub(r'[<>:"\\|?*]', "_", seg)
seg = seg.strip(" .")
if not seg:
seg = "_"
if len(seg) > 90:
seg = seg[:80] + "_" + hashlib.md5(seg.encode()).hexdigest()[:8]
return seg
def local_path(u, is_page):
p = urlparse(u)
path = unquote(p.path)
if path.endswith("/"):
path += "index"
segs = [sanitize_seg(s) for s in path.split("/") if s]
if not segs:
segs = ["index"]
name = segs.pop()
if is_page:
name += ".html"
else:
if "." not in name or not re.search(r"\.[A-Za-z0-9]{1,6}$", name):
name += ".bin"
if p.query:
qs = sanitize_seg(unquote(p.query).replace("/", "_").replace("\\", "_"))[:120]
name = name.rsplit(".", 1)[0] + "@@" + qs + "." + name.rsplit(".", 1)[1]
return os.path.join(*segs, name)
def http_get(u, timeout=30):
"""urllib GET，返回 (bytes, content_type)。处理 gzip/deflate。"""
req = Request(u, headers={"User-Agent": UA, "Accept-Encoding": "identity"})
resp = urlopen(req, timeout=timeout)
data = resp.read()
headers = resp.headers
enc = (headers.get("Content-Encoding", "") or "").lower()
if enc == "gzip":
try:
data = gzip.decompress(data)
except Exception:
pass
elif enc == "deflate":
try:
data = zlib.decompress(data)
except Exception:
pass
ct = headers.get("Content-Type", "") or ""
return data, ct
def fetch(u, timeout=(15, 60)):
for attempt in range(3):
try:
return http_get(u, timeout=timeout[1])
except HTTPError as e:
if e.code in (400, 401, 403, 404, 410):
raise
if attempt == 2:
raise
time.sleep(2 * (attempt + 1))
except Exception:
if attempt == 2:
raise
time.sleep(2 * (attempt + 1))
def decode(data, ct):
"""字节 -> 文本，尽力识别编码。"""
enc = None
m = re.search(r'charset=["\']?([\w\-]+)', ct, re.I)
if m:
enc = m.group(1)
if not enc:
head = data[:4096]
m = re.search(br'<meta[^>]+charset=["\']?([\w\-]+)', head, re.I)
if m:
enc = m.group(1).decode("ascii", "ignore")
if not enc and data.startswith(b"\xef\xbb\xbf"):
enc = "utf-8-sig"
if not enc:
enc = "utf-8"
for e in (enc, "utf-8", "gbk", "gb18030", "latin-1"):
try:
return data.decode(e)
except Exception:
continue
return data.decode("latin-1", "replace")
def download_file(u, kind):
"""下载二进制资源/附件，返回相对路径。"""
if skip_asset(u):
return None
rel = local_path(u, is_page=False)
fp = os.path.join(OUT, rel)
with LOCK:
if u in manifest and manifest[u].get("status") == "ok" and os.path.exists(fp):
return rel
manifest[u] = {"path": rel, "type": kind, "status": "pending"}
save_manifest()
try:
data, ct = http_get(u)
os.makedirs(os.path.dirname(fp), exist_ok=True)
with open(fp, "wb") as f:
f.write(data)
# CSS: 改写 url() 引用为本地路径
if kind == "asset" and (rel.lower().endswith(".css") or "css" in ct):
try:
new_css = css_urls(data.decode("utf-8", "replace"), u)
with open(fp, "w", encoding="utf-8") as f:
f.write(new_css)
except Exception:
pass
with LOCK:
manifest[u] = {"path": rel, "type": kind, "status": "ok",
"size": len(data), "ct": ct}
save_manifest()
return rel
except HTTPError as e:
with LOCK:
failed.append((u, str(e)))
manifest[u] = {"path": rel, "type": kind,
"status": "dead" if e.code in (404, 410) else "failed", "err": str(e)[:200]}
save_manifest()
if u not in failed_logged:
failed_logged.add(u)
log(f" [FAIL] {kind} {u} : {e}")
return None
except Exception as e:
with LOCK:
failed.append((u, str(e)))
manifest[u] = {"path": rel, "type": kind, "status": "failed", "err": str(e)[:200]}
save_manifest()
if u not in failed_logged:
failed_logged.add(u)
log(f" [FAIL] {kind} {u} : {e}")
return None
def css_urls(css, css_url):
def rep(m):
raw = m.group(1).strip().strip("'\"")
if raw.startswith(("data:", "javascript:", "#")):
return m.group(0)
if raw.startswith("//"):
absu = "http:" + raw
else:
absu = urljoin(css_url, raw)
rel = download_file(absu, "asset")
if rel:
from_dir = os.path.dirname(local_path(css_url, is_page=False))
rr = os.path.relpath(rel, from_dir).replace("\\", "/")
return f"url({rr})"
return m.group(0)
return re.sub(r"url\(\s*(.*?)\s*\)", rep, css, flags=re.I | re.S)
def process_page(u, data, ct):
html = decode(data, ct)
page_rel = local_path(u, is_page=True)
page_dir = os.path.dirname(page_rel)
# base 标签：用于解析相对链接，随后删除
base_for_resolve = u
m = re.search(r'<base\b[^>]*href\s*=\s*(["\'])(.*?)\1', html, re.I | re.S)
if m:
base_for_resolve = urljoin(u, m.group(2).strip())
html = re.sub(r'<base\b[^>]*/?>', '', html, flags=re.I)
new_pages, atts, assets = [], [], []
def to_local(absu, is_page):
rel = local_path(absu, is_page=is_page)
return os.path.relpath(rel, page_dir).replace("\\", "/")
def handle_href(raw):
v = raw.strip()
if not v or v.startswith(("#", "javascript:", "mailto:", "tel:", "data:", "about:")):
return raw
frag = ""
if "#" in v:
v, _, fr = v.partition("#")
frag = "#" + fr
absu = norm_url(urljoin(base_for_resolve, v))
if not absu:
return raw
host = (urlparse(absu).netloc or "").lower()
if is_attachment(absu):
atts.append(absu); rel = local_path(absu, False)
elif is_static(absu):
assets.append(absu); rel = local_path(absu, False)
elif host in PAGE_HOSTS:
new_pages.append(absu); rel = local_path(absu, True)
else:
ext_html_seen.add(absu); return raw
return to_local(absu, is_page=not (is_attachment(absu) or is_static(absu))) + frag
def handle_src(raw):
v = raw.strip()
if not v or v.startswith(("data:", "javascript:", "about:")):
return raw
absu = norm_url(urljoin(base_for_resolve, v))
if not absu:
return raw
assets.append(absu)
return to_local(absu, False)
# href / src / data / poster 属性
def sub_attr(html, attr, handler):
pat = re.compile(r'\b(' + attr + r')\s*=\s*(["\'])(.*?)\2', re.I | re.S)
return pat.sub(lambda m: m.group(1) + "=" + m.group(2) + handler(m.group(3)) + m.group(2), html)
html = sub_attr(html, r'href', handle_href)
html = sub_attr(html, r'src', handle_src)
html = sub_attr(html, r'data', handle_src)
html = sub_attr(html, r'poster', handle_src)
# srcset（逗号分隔的 URL + 描述符列表）
def sub_srcset(html):
def rep(m):
parts = []
for part in m.group(3).split(","):
part = part.strip()
if not part:
continue
toks = part.split()
su = toks[0]
rest = " ".join(toks[1:])
newu = handle_src(su)
parts.append(newu + (" " + rest if rest else ""))
return m.group(1) + "=" + m.group(2) + ", ".join(parts) + m.group(2)
pat = re.compile(r'\b(srcset)\s*=\s*(["\'])(.*?)\2', re.I | re.S)
return pat.sub(rep, html)
html = sub_srcset(html)
# data-* 属性：仅提取 URL 用于发现页面，不改写（避免破坏 JS）
for m in re.finditer(r'\b(data-[a-z0-9_-]+)\s*=\s*(["\'])(.*?)\2', html, re.I | re.S):
v = m.group(3).strip()
if not (v.startswith("/") or v.startswith("http") or re.search(r"\.(html?|php|cfm|aspx|jsp)\b", v)):
continue
absu = norm_url(urljoin(base_for_resolve, v))
if not absu:
continue
host = (urlparse(absu).netloc or "").lower()
if host in PAGE_HOSTS and not is_attachment(absu) and not is_junk_page(absu):
new_pages.append(absu)
# 确保有 charset meta
if not re.search(r'<meta[^>]+charset', html, re.I):
m = re.search(r'<head\b[^>]*>', html, re.I)
if m:
html = html[:m.end()] + '<meta charset="utf-8">' + html[m.end():]
fp = os.path.join(OUT, page_rel)
os.makedirs(os.path.dirname(fp), exist_ok=True)
with open(fp, "w", encoding="utf-8") as f:
f.write(html)
manifest[u] = {"path": page_rel, "type": "page", "status": "ok", "size": len(data)}
save_manifest()
return new_pages, atts, assets
def main():
q = deque()
for u in load_q() + [norm_url(u) for u in START_URLS] + \
[u for u, v in manifest.items() if v.get("type") == "page" and v.get("status") in ("failed", "pending")]:
u = norm_url(u)
if u and u not in q and not is_junk_page(u):
q.append(u)
log(f"队列 {len(q)} 个页面待处理")
pages_done = 0
pend = [(u, manifest[u].get("type", "asset")) for u, v in manifest.items()
if v.get("type") in ("asset", "attachment") and v.get("status") in ("pending", "failed")]
if pend:
log(f"续传未完成文件 {len(pend)} 个")
for u, k in pend:
POOL.submit(download_file, u, k)
while q:
u = q.popleft()
if u in visited:
continue
visited.add(u)
if is_junk_page(u):
manifest[u] = {"type": "page", "status": "dead", "err": "junk"}
continue
try:
data, ct = fetch(u)
except HTTPError as e:
failed.append((u, str(e)))
manifest[u] = {"type": "page", "status": "dead" if e.code in (404, 410) else "failed", "err": str(e)[:200]}
save_manifest()
if u not in failed_logged:
failed_logged.add(u)
log(f"[FAIL] page {u} : {e}")
time.sleep(DELAY)
continue
except Exception as e:
failed.append((u, str(e)))
manifest[u] = {"type": "page", "status": "failed", "err": str(e)[:200]}
save_manifest()
if u not in failed_logged:
failed_logged.add(u)
log(f"[FAIL] page {u} : {e}")
time.sleep(DELAY)
continue
pth = urlparse(u).path.lower()
if "html" in ct.lower() or pth.endswith((".htm", ".html", ".cfm", ".php", ".aspx", ".jsp")):
try:
new_pages, atts, assets = process_page(u, data, ct)
pages_done += 1
for a in atts:
POOL.submit(download_file, a, "attachment")
for a in assets:
POOL.submit(download_file, a, "asset")
for p in new_pages:
if p not in visited and p not in q:
q.append(p)
save_q(q)
if pages_done % 25 == 0:
log(f"... 已处理 {pages_done} 页, 队列 {len(q)}, 文件 {len(manifest)}")
except Exception as e:
log(f"[ERR] parse {u}: {e}")
else:
download_file(u, "attachment" if is_attachment(u) else "asset")
if pages_done > MAX_PAGES:
log("达到页面数上限，停止")
break
time.sleep(DELAY)
POOL.shutdown(wait=True)
log("=" * 60)
log(f"完成。页面 {pages_done} 个，manifest 条目 {len(manifest)} 个")
types = {}
for v in manifest.values():
types[v.get("type", "?")] = types.get(v.get("type", "?"), 0) + 1
log("类型统计:", types)
if failed:
log(f"失败 {len(failed)} 个:")
for u, e in failed:
log(" ", u, "|", e)
if ext_html_seen:
log(f"未跟进的站外 HTML 链接 {len(ext_html_seen)} 个 (示例):")
for u in list(ext_html_seen)[:10]:
log(" ", u)
if __name__ == "__main__":
log(f"=== 开始镜像 {TARGET} @ {time.strftime('%Y-%m-%d %H:%M:%S')} ===")
main()

1
blog_mirror/queue.json Normal file
View File

@@ -0,0 +1 @@
["http://171.80.3.206:39897/assets/vendor-vue-DiG09V7Q.js", "http://171.80.3.206:39897/assets/vendor-i18n-F92pr2wT.js", "http://171.80.3.206:39897/assets/vendor-misc-CbSuWkr5.js", "http://171.80.3.206:39897/assets/vendor-misc-DB0Q8XAf.css", "http://171.80.3.206:39897/assets/index-Dk4t2Wci.css"]

68
blog_mirror/run.bat Normal file
View File

@@ -0,0 +1,68 @@
@echo off
setlocal
chcp 65001 >nul
set PYTHONIOENCODING=utf-8
cd /d "%~dp0"
echo ============================================
echo Website Mirror Crawler
echo Target: http://171.80.3.206:39897/
echo ============================================
echo.
set "PY="
python --version >nul 2>nul
if not errorlevel 1 (
set "PY=python"
goto :found
)
py -3 --version >nul 2>nul
if not errorlevel 1 (
set "PY=py -3"
goto :found
)
python3 --version >nul 2>nul
if not errorlevel 1 (
set "PY=python3"
goto :found
)
if exist "C:\Users\ruigu\AppData\Local\Programs\Python\Python310\python.exe" (
set "PY=C:\Users\ruigu\AppData\Local\Programs\Python\Python310\python.exe"
goto :found
)
echo [ERROR] No Python found.
echo Install Python 3.8+ from https://www.python.org/downloads/
echo During install, check the box "Add Python to PATH".
echo Then double-click this file again.
pause
exit /b 1
:found
echo Using Python: %PY%
echo.
echo [1/3] Checking dependencies...
%PY% -c "import requests, bs4" >nul 2>nul
if errorlevel 1 (
echo [1/3] Installing requests and beautifulsoup4 ...
%PY% -m pip install -i https://pypi.tuna.tsinghua.edu.cn/simple requests beautifulsoup4
) else (
echo [1/3] Dependencies OK
)
echo.
echo [2/3] Crawling site ... this may take minutes. Do NOT close this window.
echo.
%PY% -u mirror.py http://171.80.3.206:39897/
echo.
echo [3/3] Done! Offline copy is here:
echo %~dp0mirror\171.80.3.206_39897\
echo Open index.html inside that folder with your browser.
echo.
pause

37
blog_mirror/一键抓取.bat Normal file
View File

@@ -0,0 +1,37 @@
@echo off
chcp 65001 >nul
cd /d "%~dp0"
echo ============================================
echo 网站镜像抓取脚本
echo 目标: http://171.80.3.206:39897/
echo ============================================
echo.
where python >nul 2>nul
if errorlevel 1 (
echo [错误] 没找到 python，请先安装 Python 3.8+
pause
exit /b 1
)
echo [1/3] 检查依赖...
python -c "import requests, bs4" >nul 2>nul
if errorlevel 1 (
echo [1/3] 缺少依赖，正在安装 requests beautifulsoup4 ...
python -m pip install -i https://pypi.tuna.tsinghua.edu.cn/simple requests beautifulsoup4
) else (
echo [1/3] 依赖已就绪
)
echo.
echo [2/3] 开始抓取（会持续几分钟到几十分钟，请勿关闭窗口）...
echo.
python -u mirror.py http://171.80.3.206:39897/
echo.
echo [3/3] 完成！离线站点在 mirror 目录下：
echo %~dp0mirror\171.80.3.206_39897\
echo 入口文件：用浏览器打开上面的 index.html
echo.
pause