130 lines
4.0 KiB
Python
130 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Download all unique catalog image blobs into ./images/ (local only)."""
|
|
from __future__ import annotations
|
|
|
|
import concurrent.futures
|
|
import ssl
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
IDS_FILE = ROOT / "image-ids.txt"
|
|
OUT_DIR = ROOT / "images"
|
|
CDN = "https://assets.joqhl8w6.on-eva.io/blob/"
|
|
WORKERS = 12
|
|
RETRIES = 3
|
|
UA = "KikomikoLocalMirror/1.0"
|
|
|
|
CTX = ssl._create_unverified_context()
|
|
|
|
|
|
def already_have(iid: str) -> Path | None:
|
|
for ext in (".webp", ".jpg", ".jpeg", ".png", ".gif", ".bin"):
|
|
p = OUT_DIR / f"{iid}{ext}"
|
|
if p.is_file() and p.stat().st_size > 0:
|
|
return p
|
|
return None
|
|
|
|
|
|
def ext_for(content_type: str | None, data: bytes) -> str:
|
|
ct = (content_type or "").split(";")[0].strip().lower()
|
|
if ct in ("image/webp",):
|
|
return ".webp"
|
|
if ct in ("image/jpeg", "image/jpg"):
|
|
return ".jpg"
|
|
if ct in ("image/png",):
|
|
return ".png"
|
|
if ct in ("image/gif",):
|
|
return ".gif"
|
|
if data[:3] == b"\xff\xd8\xff":
|
|
return ".jpg"
|
|
if data[:8] == b"\x89PNG\r\n\x1a\n":
|
|
return ".png"
|
|
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
|
return ".webp"
|
|
if data[:6] in (b"GIF87a", b"GIF89a"):
|
|
return ".gif"
|
|
return ".bin"
|
|
|
|
|
|
def download_one(iid: str) -> tuple[str, str, int]:
|
|
if already_have(iid):
|
|
return iid, "skip", 0
|
|
url = CDN + iid
|
|
last_err = ""
|
|
for attempt in range(1, RETRIES + 1):
|
|
try:
|
|
req = urllib.request.Request(
|
|
url,
|
|
headers={"User-Agent": UA, "Accept": "image/*,*/*"},
|
|
)
|
|
with urllib.request.urlopen(req, timeout=60, context=CTX) as resp:
|
|
data = resp.read()
|
|
if not data:
|
|
raise RuntimeError("empty body")
|
|
ext = ext_for(resp.headers.get("Content-Type"), data)
|
|
dest = OUT_DIR / f"{iid}{ext}"
|
|
tmp = OUT_DIR / f".{iid}.{ext}.partial"
|
|
tmp.write_bytes(data)
|
|
tmp.replace(dest)
|
|
return iid, "ok", len(data)
|
|
except Exception as exc: # noqa: BLE001
|
|
last_err = f"{type(exc).__name__}: {exc}"
|
|
time.sleep(0.4 * attempt)
|
|
return iid, f"fail:{last_err}", 0
|
|
|
|
|
|
def main() -> int:
|
|
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
ids = [line.strip().lower() for line in IDS_FILE.read_text().splitlines() if line.strip()]
|
|
total = len(ids)
|
|
print(f"Downloading {total} images → {OUT_DIR}", flush=True)
|
|
|
|
done = ok = skip = fail = 0
|
|
bytes_ok = 0
|
|
t0 = time.time()
|
|
fails: list[str] = []
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
|
futures = {pool.submit(download_one, iid): iid for iid in ids}
|
|
for fut in concurrent.futures.as_completed(futures):
|
|
iid, status, nbytes = fut.result()
|
|
done += 1
|
|
if status == "ok":
|
|
ok += 1
|
|
bytes_ok += nbytes
|
|
elif status == "skip":
|
|
skip += 1
|
|
else:
|
|
fail += 1
|
|
fails.append(f"{iid}\t{status}")
|
|
if done % 50 == 0 or done == total:
|
|
elapsed = max(time.time() - t0, 0.001)
|
|
rate = done / elapsed
|
|
print(
|
|
f"[{done}/{total}] ok={ok} skip={skip} fail={fail} "
|
|
f"{bytes_ok/1e6:.1f}MB new {rate:.1f}/s",
|
|
flush=True,
|
|
)
|
|
|
|
fail_path = ROOT / "download-failures.txt"
|
|
if fails:
|
|
fail_path.write_text("\n".join(fails) + "\n")
|
|
elif fail_path.exists():
|
|
fail_path.unlink()
|
|
|
|
elapsed = time.time() - t0
|
|
print(
|
|
f"Done in {elapsed/60:.1f}m — ok={ok} skip={skip} fail={fail} "
|
|
f"new_bytes={bytes_ok/1e9:.2f}GB → {OUT_DIR}",
|
|
flush=True,
|
|
)
|
|
return 1 if fail and ok == 0 else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|