first deployment of kikomiko with url photos
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
shayan khoshrou
co-authored by
Cursor
Cursor
parent
8d98304137
commit
ae3ed7c7b7
@@ -0,0 +1,9 @@
|
||||
# Local product images
|
||||
|
||||
Downloaded catalog blobs live in `images/` (gitignored).
|
||||
|
||||
```bash
|
||||
python3 download_images.py
|
||||
```
|
||||
|
||||
`serve.py` and Docker nginx serve `/i/{uuid}` from `images/` first, then fall back to Eva CDN.
|
||||
@@ -0,0 +1,170 @@
|
||||
Downloading 8362 images → /Users/shayankhoshrou/Desktop/kikomiko-frontend/exports/kikomiko-importer-repaired/kikomiko/kikomiko/media/images
|
||||
[50/8362] ok=50 skip=0 fail=0 10.0MB new 13.2/s
|
||||
[100/8362] ok=100 skip=0 fail=0 19.5MB new 12.8/s
|
||||
[150/8362] ok=150 skip=0 fail=0 32.6MB new 13.3/s
|
||||
[200/8362] ok=200 skip=0 fail=0 46.4MB new 12.8/s
|
||||
[250/8362] ok=250 skip=0 fail=0 58.7MB new 12.9/s
|
||||
[300/8362] ok=300 skip=0 fail=0 66.6MB new 13.4/s
|
||||
[350/8362] ok=350 skip=0 fail=0 77.6MB new 13.8/s
|
||||
[400/8362] ok=400 skip=0 fail=0 90.5MB new 13.7/s
|
||||
[450/8362] ok=450 skip=0 fail=0 105.4MB new 13.2/s
|
||||
[500/8362] ok=500 skip=0 fail=0 114.7MB new 13.4/s
|
||||
[550/8362] ok=550 skip=0 fail=0 128.7MB new 13.3/s
|
||||
[600/8362] ok=600 skip=0 fail=0 142.2MB new 13.0/s
|
||||
[650/8362] ok=650 skip=0 fail=0 152.8MB new 13.2/s
|
||||
[700/8362] ok=700 skip=0 fail=0 162.3MB new 13.5/s
|
||||
[750/8362] ok=750 skip=0 fail=0 169.4MB new 13.8/s
|
||||
[800/8362] ok=800 skip=0 fail=0 176.4MB new 14.0/s
|
||||
[850/8362] ok=850 skip=0 fail=0 187.6MB new 14.0/s
|
||||
[900/8362] ok=900 skip=0 fail=0 195.6MB new 14.1/s
|
||||
[950/8362] ok=950 skip=0 fail=0 203.8MB new 14.2/s
|
||||
[1000/8362] ok=1000 skip=0 fail=0 211.1MB new 14.3/s
|
||||
[1050/8362] ok=1050 skip=0 fail=0 223.9MB new 14.2/s
|
||||
[1100/8362] ok=1100 skip=0 fail=0 237.8MB new 14.1/s
|
||||
[1150/8362] ok=1150 skip=0 fail=0 246.1MB new 14.2/s
|
||||
[1200/8362] ok=1200 skip=0 fail=0 256.5MB new 14.2/s
|
||||
[1250/8362] ok=1250 skip=0 fail=0 262.6MB new 14.4/s
|
||||
[1300/8362] ok=1300 skip=0 fail=0 280.1MB new 14.0/s
|
||||
[1350/8362] ok=1350 skip=0 fail=0 293.0MB new 13.9/s
|
||||
[1400/8362] ok=1400 skip=0 fail=0 302.9MB new 14.0/s
|
||||
[1450/8362] ok=1450 skip=0 fail=0 312.0MB new 14.0/s
|
||||
[1500/8362] ok=1500 skip=0 fail=0 322.5MB new 14.1/s
|
||||
[1550/8362] ok=1550 skip=0 fail=0 329.9MB new 14.2/s
|
||||
[1600/8362] ok=1600 skip=0 fail=0 341.0MB new 14.2/s
|
||||
[1650/8362] ok=1650 skip=0 fail=0 350.2MB new 14.2/s
|
||||
[1700/8362] ok=1700 skip=0 fail=0 362.3MB new 14.1/s
|
||||
[1750/8362] ok=1750 skip=0 fail=0 373.0MB new 14.2/s
|
||||
[1800/8362] ok=1800 skip=0 fail=0 381.3MB new 14.3/s
|
||||
[1850/8362] ok=1850 skip=0 fail=0 389.4MB new 14.3/s
|
||||
[1900/8362] ok=1900 skip=0 fail=0 401.3MB new 14.2/s
|
||||
[1950/8362] ok=1950 skip=0 fail=0 414.0MB new 14.2/s
|
||||
[2000/8362] ok=2000 skip=0 fail=0 425.1MB new 14.2/s
|
||||
[2050/8362] ok=2050 skip=0 fail=0 434.4MB new 14.3/s
|
||||
[2100/8362] ok=2100 skip=0 fail=0 443.9MB new 14.3/s
|
||||
[2150/8362] ok=2150 skip=0 fail=0 452.5MB new 14.4/s
|
||||
[2200/8362] ok=2200 skip=0 fail=0 461.6MB new 14.4/s
|
||||
[2250/8362] ok=2250 skip=0 fail=0 472.4MB new 14.4/s
|
||||
[2300/8362] ok=2300 skip=0 fail=0 483.1MB new 14.4/s
|
||||
[2350/8362] ok=2350 skip=0 fail=0 491.0MB new 14.4/s
|
||||
[2400/8362] ok=2400 skip=0 fail=0 499.4MB new 14.5/s
|
||||
[2450/8362] ok=2450 skip=0 fail=0 506.4MB new 14.5/s
|
||||
[2500/8362] ok=2500 skip=0 fail=0 514.1MB new 14.6/s
|
||||
[2550/8362] ok=2550 skip=0 fail=0 522.3MB new 14.7/s
|
||||
[2600/8362] ok=2600 skip=0 fail=0 536.6MB new 14.4/s
|
||||
[2650/8362] ok=2650 skip=0 fail=0 544.1MB new 14.2/s
|
||||
[2700/8362] ok=2700 skip=0 fail=0 555.8MB new 14.1/s
|
||||
[2750/8362] ok=2750 skip=0 fail=0 566.7MB new 14.1/s
|
||||
[2800/8362] ok=2800 skip=0 fail=0 578.5MB new 14.1/s
|
||||
[2850/8362] ok=2850 skip=0 fail=0 589.1MB new 14.1/s
|
||||
[2900/8362] ok=2900 skip=0 fail=0 599.4MB new 14.1/s
|
||||
[2950/8362] ok=2950 skip=0 fail=0 608.5MB new 14.1/s
|
||||
[3000/8362] ok=3000 skip=0 fail=0 619.9MB new 14.1/s
|
||||
[3050/8362] ok=3050 skip=0 fail=0 630.3MB new 14.1/s
|
||||
[3100/8362] ok=3100 skip=0 fail=0 644.2MB new 14.1/s
|
||||
[3150/8362] ok=3150 skip=0 fail=0 652.5MB new 14.1/s
|
||||
[3200/8362] ok=3200 skip=0 fail=0 662.2MB new 14.2/s
|
||||
[3250/8362] ok=3250 skip=0 fail=0 671.3MB new 14.2/s
|
||||
[3300/8362] ok=3300 skip=0 fail=0 683.3MB new 14.2/s
|
||||
[3350/8362] ok=3350 skip=0 fail=0 693.2MB new 14.2/s
|
||||
[3400/8362] ok=3400 skip=0 fail=0 701.8MB new 14.2/s
|
||||
[3450/8362] ok=3450 skip=0 fail=0 712.1MB new 14.3/s
|
||||
[3500/8362] ok=3500 skip=0 fail=0 721.8MB new 14.3/s
|
||||
[3550/8362] ok=3550 skip=0 fail=0 731.7MB new 14.3/s
|
||||
[3600/8362] ok=3600 skip=0 fail=0 744.2MB new 14.3/s
|
||||
[3650/8362] ok=3650 skip=0 fail=0 754.0MB new 14.2/s
|
||||
[3700/8362] ok=3700 skip=0 fail=0 766.3MB new 14.2/s
|
||||
[3750/8362] ok=3750 skip=0 fail=0 776.0MB new 14.3/s
|
||||
[3800/8362] ok=3800 skip=0 fail=0 786.8MB new 14.2/s
|
||||
[3850/8362] ok=3850 skip=0 fail=0 798.9MB new 14.2/s
|
||||
[3900/8362] ok=3900 skip=0 fail=0 809.1MB new 14.2/s
|
||||
[3950/8362] ok=3950 skip=0 fail=0 819.6MB new 14.2/s
|
||||
[4000/8362] ok=4000 skip=0 fail=0 829.3MB new 14.2/s
|
||||
[4050/8362] ok=4050 skip=0 fail=0 839.2MB new 14.2/s
|
||||
[4100/8362] ok=4100 skip=0 fail=0 849.5MB new 14.2/s
|
||||
[4150/8362] ok=4150 skip=0 fail=0 858.8MB new 14.3/s
|
||||
[4200/8362] ok=4200 skip=0 fail=0 865.2MB new 14.3/s
|
||||
[4250/8362] ok=4250 skip=0 fail=0 875.8MB new 14.3/s
|
||||
[4300/8362] ok=4300 skip=0 fail=0 885.9MB new 14.3/s
|
||||
[4350/8362] ok=4350 skip=0 fail=0 893.3MB new 14.4/s
|
||||
[4400/8362] ok=4400 skip=0 fail=0 903.1MB new 14.4/s
|
||||
[4450/8362] ok=4450 skip=0 fail=0 913.2MB new 14.4/s
|
||||
[4500/8362] ok=4500 skip=0 fail=0 923.6MB new 14.4/s
|
||||
[4550/8362] ok=4550 skip=0 fail=0 930.3MB new 14.5/s
|
||||
[4600/8362] ok=4600 skip=0 fail=0 939.5MB new 14.5/s
|
||||
[4650/8362] ok=4650 skip=0 fail=0 952.7MB new 14.4/s
|
||||
[4700/8362] ok=4700 skip=0 fail=0 965.3MB new 14.4/s
|
||||
[4750/8362] ok=4750 skip=0 fail=0 976.0MB new 14.4/s
|
||||
[4800/8362] ok=4800 skip=0 fail=0 992.7MB new 14.4/s
|
||||
[4850/8362] ok=4850 skip=0 fail=0 1002.2MB new 14.4/s
|
||||
[4900/8362] ok=4900 skip=0 fail=0 1009.9MB new 14.4/s
|
||||
[4950/8362] ok=4950 skip=0 fail=0 1018.6MB new 14.4/s
|
||||
[5000/8362] ok=5000 skip=0 fail=0 1028.1MB new 14.4/s
|
||||
[5050/8362] ok=5050 skip=0 fail=0 1039.3MB new 14.4/s
|
||||
[5100/8362] ok=5100 skip=0 fail=0 1050.0MB new 14.5/s
|
||||
[5150/8362] ok=5150 skip=0 fail=0 1056.8MB new 14.5/s
|
||||
[5200/8362] ok=5200 skip=0 fail=0 1062.7MB new 14.5/s
|
||||
[5250/8362] ok=5250 skip=0 fail=0 1070.3MB new 14.5/s
|
||||
[5300/8362] ok=5300 skip=0 fail=0 1081.8MB new 14.4/s
|
||||
[5350/8362] ok=5350 skip=0 fail=0 1092.1MB new 14.4/s
|
||||
[5400/8362] ok=5400 skip=0 fail=0 1100.2MB new 14.4/s
|
||||
[5450/8362] ok=5450 skip=0 fail=0 1108.0MB new 14.4/s
|
||||
[5500/8362] ok=5500 skip=0 fail=0 1117.8MB new 14.5/s
|
||||
[5550/8362] ok=5550 skip=0 fail=0 1127.6MB new 14.4/s
|
||||
[5600/8362] ok=5600 skip=0 fail=0 1140.1MB new 14.4/s
|
||||
[5650/8362] ok=5650 skip=0 fail=0 1154.1MB new 14.3/s
|
||||
[5700/8362] ok=5700 skip=0 fail=0 1165.3MB new 14.3/s
|
||||
[5750/8362] ok=5750 skip=0 fail=0 1177.7MB new 14.3/s
|
||||
[5800/8362] ok=5800 skip=0 fail=0 1190.1MB new 14.2/s
|
||||
[5850/8362] ok=5850 skip=0 fail=0 1201.4MB new 14.2/s
|
||||
[5900/8362] ok=5900 skip=0 fail=0 1210.8MB new 14.2/s
|
||||
[5950/8362] ok=5950 skip=0 fail=0 1218.5MB new 14.3/s
|
||||
[6000/8362] ok=6000 skip=0 fail=0 1230.4MB new 14.2/s
|
||||
[6050/8362] ok=6050 skip=0 fail=0 1242.5MB new 14.2/s
|
||||
[6100/8362] ok=6100 skip=0 fail=0 1251.9MB new 14.1/s
|
||||
[6150/8362] ok=6150 skip=0 fail=0 1263.2MB new 14.1/s
|
||||
[6200/8362] ok=6200 skip=0 fail=0 1270.2MB new 14.1/s
|
||||
[6250/8362] ok=6250 skip=0 fail=0 1276.2MB new 14.1/s
|
||||
[6300/8362] ok=6300 skip=0 fail=0 1290.4MB new 14.0/s
|
||||
[6350/8362] ok=6350 skip=0 fail=0 1298.9MB new 14.1/s
|
||||
[6400/8362] ok=6400 skip=0 fail=0 1307.8MB new 14.0/s
|
||||
[6450/8362] ok=6450 skip=0 fail=0 1315.9MB new 14.1/s
|
||||
[6500/8362] ok=6500 skip=0 fail=0 1325.3MB new 14.1/s
|
||||
[6550/8362] ok=6550 skip=0 fail=0 1335.1MB new 14.1/s
|
||||
[6600/8362] ok=6600 skip=0 fail=0 1343.8MB new 14.1/s
|
||||
[6650/8362] ok=6650 skip=0 fail=0 1355.6MB new 14.1/s
|
||||
[6700/8362] ok=6700 skip=0 fail=0 1367.1MB new 14.1/s
|
||||
[6750/8362] ok=6750 skip=0 fail=0 1377.7MB new 14.1/s
|
||||
[6800/8362] ok=6800 skip=0 fail=0 1390.2MB new 14.1/s
|
||||
[6850/8362] ok=6850 skip=0 fail=0 1402.2MB new 14.1/s
|
||||
[6900/8362] ok=6900 skip=0 fail=0 1410.9MB new 14.1/s
|
||||
[6950/8362] ok=6950 skip=0 fail=0 1423.6MB new 14.1/s
|
||||
[7000/8362] ok=7000 skip=0 fail=0 1430.6MB new 14.1/s
|
||||
[7050/8362] ok=7050 skip=0 fail=0 1445.8MB new 14.1/s
|
||||
[7100/8362] ok=7100 skip=0 fail=0 1457.6MB new 14.1/s
|
||||
[7150/8362] ok=7150 skip=0 fail=0 1466.2MB new 14.1/s
|
||||
[7200/8362] ok=7200 skip=0 fail=0 1475.2MB new 14.1/s
|
||||
[7250/8362] ok=7250 skip=0 fail=0 1483.1MB new 14.1/s
|
||||
[7300/8362] ok=7300 skip=0 fail=0 1494.0MB new 14.2/s
|
||||
[7350/8362] ok=7350 skip=0 fail=0 1502.2MB new 14.2/s
|
||||
[7400/8362] ok=7400 skip=0 fail=0 1517.2MB new 14.2/s
|
||||
[7450/8362] ok=7450 skip=0 fail=0 1524.3MB new 14.2/s
|
||||
[7500/8362] ok=7500 skip=0 fail=0 1532.0MB new 14.2/s
|
||||
[7550/8362] ok=7550 skip=0 fail=0 1544.9MB new 14.2/s
|
||||
[7600/8362] ok=7600 skip=0 fail=0 1552.5MB new 14.2/s
|
||||
[7650/8362] ok=7650 skip=0 fail=0 1563.5MB new 14.2/s
|
||||
[7700/8362] ok=7700 skip=0 fail=0 1573.0MB new 14.2/s
|
||||
[7750/8362] ok=7750 skip=0 fail=0 1588.4MB new 14.2/s
|
||||
[7800/8362] ok=7800 skip=0 fail=0 1598.7MB new 14.2/s
|
||||
[7850/8362] ok=7850 skip=0 fail=0 1607.2MB new 14.2/s
|
||||
[7900/8362] ok=7900 skip=0 fail=0 1614.8MB new 14.2/s
|
||||
[7950/8362] ok=7950 skip=0 fail=0 1622.8MB new 14.3/s
|
||||
[8000/8362] ok=8000 skip=0 fail=0 1630.9MB new 14.3/s
|
||||
[8050/8362] ok=8050 skip=0 fail=0 1642.6MB new 14.3/s
|
||||
[8100/8362] ok=8100 skip=0 fail=0 1651.9MB new 14.3/s
|
||||
[8150/8362] ok=8150 skip=0 fail=0 1664.4MB new 14.3/s
|
||||
[8200/8362] ok=8200 skip=0 fail=0 1675.2MB new 14.3/s
|
||||
[8250/8362] ok=8250 skip=0 fail=0 1682.3MB new 14.3/s
|
||||
[8300/8362] ok=8300 skip=0 fail=0 1692.5MB new 14.3/s
|
||||
[8350/8362] ok=8350 skip=0 fail=0 1701.7MB new 14.3/s
|
||||
[8362/8362] ok=8362 skip=0 fail=0 1705.0MB new 14.3/s
|
||||
Done in 9.7m — ok=8362 skip=0 fail=0 new_bytes=1.71GB → /Users/shayankhoshrou/Desktop/kikomiko-frontend/exports/kikomiko-importer-repaired/kikomiko/kikomiko/media/images
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download all unique catalog image blobs into ./images/ (local only)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
IDS_FILE = ROOT / "image-ids.txt"
|
||||
OUT_DIR = ROOT / "images"
|
||||
CDN = "https://assets.joqhl8w6.on-eva.io/blob/"
|
||||
WORKERS = 12
|
||||
RETRIES = 3
|
||||
UA = "KikomikoLocalMirror/1.0"
|
||||
|
||||
CTX = ssl._create_unverified_context()
|
||||
|
||||
|
||||
def already_have(iid: str) -> Path | None:
|
||||
for ext in (".webp", ".jpg", ".jpeg", ".png", ".gif", ".bin"):
|
||||
p = OUT_DIR / f"{iid}{ext}"
|
||||
if p.is_file() and p.stat().st_size > 0:
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def ext_for(content_type: str | None, data: bytes) -> str:
|
||||
ct = (content_type or "").split(";")[0].strip().lower()
|
||||
if ct in ("image/webp",):
|
||||
return ".webp"
|
||||
if ct in ("image/jpeg", "image/jpg"):
|
||||
return ".jpg"
|
||||
if ct in ("image/png",):
|
||||
return ".png"
|
||||
if ct in ("image/gif",):
|
||||
return ".gif"
|
||||
if data[:3] == b"\xff\xd8\xff":
|
||||
return ".jpg"
|
||||
if data[:8] == b"\x89PNG\r\n\x1a\n":
|
||||
return ".png"
|
||||
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
return ".webp"
|
||||
if data[:6] in (b"GIF87a", b"GIF89a"):
|
||||
return ".gif"
|
||||
return ".bin"
|
||||
|
||||
|
||||
def download_one(iid: str) -> tuple[str, str, int]:
|
||||
if already_have(iid):
|
||||
return iid, "skip", 0
|
||||
url = CDN + iid
|
||||
last_err = ""
|
||||
for attempt in range(1, RETRIES + 1):
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"User-Agent": UA, "Accept": "image/*,*/*"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60, context=CTX) as resp:
|
||||
data = resp.read()
|
||||
if not data:
|
||||
raise RuntimeError("empty body")
|
||||
ext = ext_for(resp.headers.get("Content-Type"), data)
|
||||
dest = OUT_DIR / f"{iid}{ext}"
|
||||
tmp = OUT_DIR / f".{iid}.{ext}.partial"
|
||||
tmp.write_bytes(data)
|
||||
tmp.replace(dest)
|
||||
return iid, "ok", len(data)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_err = f"{type(exc).__name__}: {exc}"
|
||||
time.sleep(0.4 * attempt)
|
||||
return iid, f"fail:{last_err}", 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
ids = [line.strip().lower() for line in IDS_FILE.read_text().splitlines() if line.strip()]
|
||||
total = len(ids)
|
||||
print(f"Downloading {total} images → {OUT_DIR}", flush=True)
|
||||
|
||||
done = ok = skip = fail = 0
|
||||
bytes_ok = 0
|
||||
t0 = time.time()
|
||||
fails: list[str] = []
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
||||
futures = {pool.submit(download_one, iid): iid for iid in ids}
|
||||
for fut in concurrent.futures.as_completed(futures):
|
||||
iid, status, nbytes = fut.result()
|
||||
done += 1
|
||||
if status == "ok":
|
||||
ok += 1
|
||||
bytes_ok += nbytes
|
||||
elif status == "skip":
|
||||
skip += 1
|
||||
else:
|
||||
fail += 1
|
||||
fails.append(f"{iid}\t{status}")
|
||||
if done % 50 == 0 or done == total:
|
||||
elapsed = max(time.time() - t0, 0.001)
|
||||
rate = done / elapsed
|
||||
print(
|
||||
f"[{done}/{total}] ok={ok} skip={skip} fail={fail} "
|
||||
f"{bytes_ok/1e6:.1f}MB new {rate:.1f}/s",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
fail_path = ROOT / "download-failures.txt"
|
||||
if fails:
|
||||
fail_path.write_text("\n".join(fails) + "\n")
|
||||
elif fail_path.exists():
|
||||
fail_path.unlink()
|
||||
|
||||
elapsed = time.time() - t0
|
||||
print(
|
||||
f"Done in {elapsed/60:.1f}m — ok={ok} skip={skip} fail={fail} "
|
||||
f"new_bytes={bytes_ok/1e9:.2f}GB → {OUT_DIR}",
|
||||
flush=True,
|
||||
)
|
||||
return 1 if fail and ok == 0 else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user