first deployment of kikomiko with url photos

Co-authored-by: Cursor <cursoragent@cursor.com>

kikomiko/media/download_images.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""Download all unique catalog image blobs into ./images/ (local only)."""
+from __future__ import annotations
+
+import concurrent.futures
+import ssl
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+IDS_FILE = ROOT / "image-ids.txt"
+OUT_DIR = ROOT / "images"
+CDN = "https://assets.joqhl8w6.on-eva.io/blob/"
+WORKERS = 12
+RETRIES = 3
+UA = "KikomikoLocalMirror/1.0"
+
+CTX = ssl._create_unverified_context()
+
+
+def already_have(iid: str) -> Path | None:
+    for ext in (".webp", ".jpg", ".jpeg", ".png", ".gif", ".bin"):
+        p = OUT_DIR / f"{iid}{ext}"
+        if p.is_file() and p.stat().st_size > 0:
+            return p
+    return None
+
+
+def ext_for(content_type: str | None, data: bytes) -> str:
+    ct = (content_type or "").split(";")[0].strip().lower()
+    if ct in ("image/webp",):
+        return ".webp"
+    if ct in ("image/jpeg", "image/jpg"):
+        return ".jpg"
+    if ct in ("image/png",):
+        return ".png"
+    if ct in ("image/gif",):
+        return ".gif"
+    if data[:3] == b"\xff\xd8\xff":
+        return ".jpg"
+    if data[:8] == b"\x89PNG\r\n\x1a\n":
+        return ".png"
+    if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
+        return ".webp"
+    if data[:6] in (b"GIF87a", b"GIF89a"):
+        return ".gif"
+    return ".bin"
+
+
+def download_one(iid: str) -> tuple[str, str, int]:
+    if already_have(iid):
+        return iid, "skip", 0
+    url = CDN + iid
+    last_err = ""
+    for attempt in range(1, RETRIES + 1):
+        try:
+            req = urllib.request.Request(
+                url,
+                headers={"User-Agent": UA, "Accept": "image/*,*/*"},
+            )
+            with urllib.request.urlopen(req, timeout=60, context=CTX) as resp:
+                data = resp.read()
+                if not data:
+                    raise RuntimeError("empty body")
+                ext = ext_for(resp.headers.get("Content-Type"), data)
+                dest = OUT_DIR / f"{iid}{ext}"
+                tmp = OUT_DIR / f".{iid}.{ext}.partial"
+                tmp.write_bytes(data)
+                tmp.replace(dest)
+                return iid, "ok", len(data)
+        except Exception as exc:  # noqa: BLE001
+            last_err = f"{type(exc).__name__}: {exc}"
+            time.sleep(0.4 * attempt)
+    return iid, f"fail:{last_err}", 0
+
+
+def main() -> int:
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    ids = [line.strip().lower() for line in IDS_FILE.read_text().splitlines() if line.strip()]
+    total = len(ids)
+    print(f"Downloading {total} images → {OUT_DIR}", flush=True)
+
+    done = ok = skip = fail = 0
+    bytes_ok = 0
+    t0 = time.time()
+    fails: list[str] = []
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=WORKERS) as pool:
+        futures = {pool.submit(download_one, iid): iid for iid in ids}
+        for fut in concurrent.futures.as_completed(futures):
+            iid, status, nbytes = fut.result()
+            done += 1
+            if status == "ok":
+                ok += 1
+                bytes_ok += nbytes
+            elif status == "skip":
+                skip += 1
+            else:
+                fail += 1
+                fails.append(f"{iid}\t{status}")
+            if done % 50 == 0 or done == total:
+                elapsed = max(time.time() - t0, 0.001)
+                rate = done / elapsed
+                print(
+                    f"[{done}/{total}] ok={ok} skip={skip} fail={fail} "
+                    f"{bytes_ok/1e6:.1f}MB new  {rate:.1f}/s",
+                    flush=True,
+                )
+
+    fail_path = ROOT / "download-failures.txt"
+    if fails:
+        fail_path.write_text("\n".join(fails) + "\n")
+    elif fail_path.exists():
+        fail_path.unlink()
+
+    elapsed = time.time() - t0
+    print(
+        f"Done in {elapsed/60:.1f}m — ok={ok} skip={skip} fail={fail} "
+        f"new_bytes={bytes_ok/1e9:.2f}GB → {OUT_DIR}",
+        flush=True,
+    )
+    return 1 if fail and ok == 0 else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())