Files
Kikomiko/kikomiko/admin/server.py
T

2090 lines
86 KiB
Python

#!/usr/bin/env python3
"""Kikomiko admin CMS — Python stdlib + SQLite. Run: python3 server.py --port 8090"""
from __future__ import annotations
import argparse
import hashlib
import hmac
import json
import math
import os
import re
import secrets
import shutil
import sqlite3
import sys
import tempfile
import time
import uuid
from http import HTTPStatus
from http.cookies import SimpleCookie
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import parse_qs, urlparse
from ai_translate import (
category_needs_fa,
collect_jobs,
load_settings as load_ai_settings,
public_settings as public_ai_settings,
resolve_category_api_key,
save_settings as save_ai_settings,
translate_category_name,
translate_product,
)
from detail_schema import (
default_catalog_detail_queries,
ensure_product_queries,
fill_details_via_serp,
missing_detail_keys,
)
SQLITE_HEADER = b"SQLite format 3\x00"
DB_UPLOAD_MAX = 80_000_000
ROOT = Path(__file__).resolve().parent
DATA_DIR = ROOT / "data"
STATIC_DIR = ROOT / "static"
DB_PATH = DATA_DIR / "kikomiko.db"
CREDENTIALS_PATH = DATA_DIR / ".admin-credentials"
SEED_CATEGORIES = ROOT / "seed_categories.json"
DIST_CATALOG = Path(
os.environ.get(
"KIKOMIKO_CATALOG_PATH",
str(ROOT.parent / "dist" / "data" / "catalog.dedupe2.json"),
)
)
SESSION_TTL = 60 * 60 * 12
LOGIN_WINDOW = 900
LOGIN_MAX_FAILS = 8
PBKDF2_ITERS = 310_000
# Persian labels for KIKO path segments / seed category names (admin + catalog).
KIKO_SEGMENT_FA: dict[str, str] = {
"MAKE-UP": "آرایش",
"EYES MAKE-UP": "چشم",
"FACE MAKE-UP": "صورت",
"LIPS MAKE-UP": "لب",
"NAILS MAKE-UP": "ناخن",
"SKIN CARE": "مراقبت پوست",
"FRAGRANCES": "عطر",
"ACCESSORIES": "اکسسوری",
"EYE PENCILS": "مداد چشم",
"EYEBROWS": "ابرو",
"EYELINER": "خط چشم",
"EYESHADOWS": "سایه چشم",
"MASCARAS": "ریمل",
"BLUSH": "رژگونه",
"BRONZERS": "برنزر",
"CONCEALERS": "کانسیلر",
"CONTOURING": "کانتور",
"FOUNDATION": "کرم پودر",
"HIGHLIGHTERS FACE": "هایلایتر",
"POWDERS": "پودر",
"PRIMERS AND FIXERS FACE": "پرایمر و فیکس",
"LIP GLOSS": "لیپ‌گلاس",
"LIP PENCILS": "مداد لب",
"LIPS MAKE UP KIT": "ست لب",
"LIPSTICK": "رژ لب",
"MARKER": "مارکر لب",
"NAIL LACQUER": "لاک ناخن",
"FACE SKIN CARE": "مراقبت صورت",
"LIP CARE": "مراقبت لب",
"OTHER SKINCARE": "مراقبت پوست",
"SUN CARE": "ضدآفتاب",
"FULL SIZE FRAG": "عطر",
"EYE ACCESSORIES": "اکسسوری چشم",
"MAKE-UP BAG": "کیف آرایشی",
"BRUSHES": "براش",
"FACE ACCESSORIES": "اکسسوری صورت",
"HAND AND FOOT ACCESSORIES": "اکسسوری دست و پا",
"SKIN CARE KIT": "ست مراقبت پوست",
"FACE MAKE-UP KIT": "ست آرایش صورت",
"EYES MAKE-UP KIT": "ست آرایش چشم",
"FACE CLEASING": "پاک‌کننده صورت",
"FACE CLEANSING": "پاک‌کننده صورت",
"MASK": "ماسک",
"ACCESSORIES KIT": "ست اکسسوری",
"HAIRCARE": "مراقبت مو",
"HAIR ACCESSORIES": "اکسسوری مو",
"SERVICE": "سرویس",
"EMPTY CONTAINER": "ظرف خالی",
"PRIMERS AND FIXERS EYES": "پرایمر و فیکس چشم",
"PRIMERS AND FIXERS LIPS": "پرایمر و فیکس لب",
"PRIMERS AND FIXERS FACE": "پرایمر و فیکس صورت",
"FIXER & THINNER": "فیکسر و تینر",
"FIXER AND THINNER": "فیکسر و تینر",
"MIXING SOLUTION": "محلول ترکیب",
"FRENCH MANICURE": "فرنچ مانیکور",
"FULL SIZE FRAG": "عطر",
"FULL SIZE HAIR": "مراقبت مو",
"MAKE UP": "آرایش",
"MAKE-UP": "آرایش",
"SKIN CARE": "مراقبت پوست",
"EYES MAKE-UP": "چشم",
"FACE MAKE-UP": "صورت",
"LIPS MAKE-UP": "لب",
"NAILS MAKE-UP": "ناخن",
"REMOVER": "پاک‌کننده",
"SAMPLE HAIR": "نمونه مو",
}
def persian_category_label(category: str | None) -> str:
"""Most specific Persian label for a KIKO category path.
Returns '' when no Persian mapping exists (never echo English as nameFa).
"""
parts = [p.strip() for p in str(category or "").split("/") if p.strip()]
for part in reversed(parts):
key = re.sub(r"\s+", " ", part).upper()
if key in KIKO_SEGMENT_FA:
return KIKO_SEGMENT_FA[key]
# Also accept slug-like tokens: face-make-up → FACE MAKE-UP
slugish = re.sub(r"[-_]+", " ", str(category or "")).strip().upper()
if slugish in KIKO_SEGMENT_FA:
return KIKO_SEGMENT_FA[slugish]
return ""
def seed_name_fa_by_slug() -> dict[str, str]:
out: dict[str, str] = {}
if SEED_CATEGORIES.is_file():
try:
for item in json.loads(SEED_CATEGORIES.read_text(encoding="utf-8")):
slug = str(item.get("slug") or "").strip()
fa = str(item.get("name_fa") or item.get("nameFa") or "").strip()
if slug and fa:
out[slug] = fa
except Exception:
pass
return out
sys.path.insert(0, str(ROOT))
from fetchers import get_fetcher, list_fetchers # noqa: E402
from kiko_enrich import ( # noqa: E402
_family_key_from_product_url,
discover_catalog_product_urls,
fetch_kiko_product,
merge_enrichment,
)
# --- password / session helpers -------------------------------------------------
def _hash_password(password: str, salt: bytes | None = None) -> str:
salt = salt or secrets.token_bytes(16)
dk = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, PBKDF2_ITERS)
return f"pbkdf2_sha256${PBKDF2_ITERS}${salt.hex()}${dk.hex()}"
def _verify_password(password: str, stored: str) -> bool:
try:
algo, iters_s, salt_hex, hash_hex = stored.split("$", 3)
if algo != "pbkdf2_sha256":
return False
iters = int(iters_s)
salt = bytes.fromhex(salt_hex)
expected = bytes.fromhex(hash_hex)
got = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iters)
return hmac.compare_digest(got, expected)
except Exception:
return False
def _now() -> int:
return int(time.time())
# --- database -------------------------------------------------------------------
class ClosingConnection(sqlite3.Connection):
def __exit__(self, *args):
try:
return super().__exit__(*args)
finally:
self.close()
def db() -> sqlite3.Connection:
DATA_DIR.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(DB_PATH, factory=ClosingConnection)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys = ON")
conn.execute("PRAGMA journal_mode = WAL")
return conn
def init_db() -> None:
with db() as conn:
conn.executescript(
"""
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY,
username TEXT NOT NULL UNIQUE COLLATE NOCASE,
password_hash TEXT NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS sessions (
token TEXT PRIMARY KEY,
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
csrf TEXT NOT NULL,
expires_at INTEGER NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS categories (
id INTEGER PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
name_fa TEXT NOT NULL DEFAULT '',
parent_slug TEXT,
sort_order INTEGER NOT NULL DEFAULT 0,
active INTEGER NOT NULL DEFAULT 1,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS products (
id TEXT PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
name_fa TEXT NOT NULL DEFAULT '',
price REAL NOT NULL DEFAULT 0,
compare_at_price REAL,
stock INTEGER NOT NULL DEFAULT 0,
hero_image TEXT,
category_slug TEXT,
category_fa TEXT NOT NULL DEFAULT '',
description TEXT NOT NULL DEFAULT '',
sku TEXT,
external_id TEXT,
source TEXT NOT NULL DEFAULT 'manual',
color_variants TEXT NOT NULL DEFAULT '[]',
active INTEGER NOT NULL DEFAULT 1,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_products_category ON products(category_slug);
CREATE INDEX IF NOT EXISTS idx_products_external ON products(external_id);
CREATE INDEX IF NOT EXISTS idx_products_sku ON products(sku);
CREATE TABLE IF NOT EXISTS login_attempts (
ip TEXT PRIMARY KEY,
fails INTEGER NOT NULL DEFAULT 0,
window_start INTEGER NOT NULL
);
"""
)
columns = {r["name"] for r in conn.execute("PRAGMA table_info(products)")}
if "content_json" not in columns:
conn.execute("ALTER TABLE products ADD COLUMN content_json TEXT NOT NULL DEFAULT '{}'")
conn.execute("CREATE TABLE IF NOT EXISTS catalog_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
ensure_admin_user()
def ensure_admin_user() -> None:
username = os.environ.get("ADMIN_USER", "admin").strip() or "admin"
password = os.environ.get("ADMIN_PASS")
with db() as conn:
row = conn.execute("SELECT id FROM users WHERE username = ?", (username,)).fetchone()
if row:
return
if not password:
password = secrets.token_urlsafe(16)
CREDENTIALS_PATH.write_text(
f"username={username}\npassword={password}\n"
"Change this password after first login.\n",
encoding="utf-8",
)
os.chmod(CREDENTIALS_PATH, 0o600)
print(
f"\n=== ADMIN CREDENTIALS (first run) ===\n"
f"username: {username}\npassword: {password}\n"
f"also saved to {CREDENTIALS_PATH}\n"
f"====================================\n",
flush=True,
)
conn.execute(
"INSERT INTO users (username, password_hash, created_at) VALUES (?, ?, ?)",
(username, _hash_password(password), _now()),
)
# --- row helpers ----------------------------------------------------------------
def category_dict(row: sqlite3.Row) -> dict:
return {
"id": row["id"],
"slug": row["slug"],
"name": row["name"],
"nameFa": row["name_fa"],
"parentSlug": row["parent_slug"],
"sortOrder": row["sort_order"],
"active": bool(row["active"]),
"createdAt": row["created_at"],
"updatedAt": row["updated_at"],
}
def product_dict(row: sqlite3.Row) -> dict:
try:
variants = json.loads(row["color_variants"] or "[]")
except json.JSONDecodeError:
variants = []
return {
**json.loads(row["content_json"] or "{}"),
"id": row["id"],
"slug": row["slug"],
"name": row["name"],
"nameFa": row["name_fa"],
"price": row["price"],
"compareAtPrice": row["compare_at_price"],
"stock": row["stock"],
"heroImage": row["hero_image"],
"category": row["category_slug"],
"categoryFa": row["category_fa"],
"description": row["description"],
"sku": row["sku"],
"externalId": row["external_id"],
"source": row["source"],
"colorVariants": variants,
"active": bool(row["active"]),
"createdAt": row["created_at"],
"updatedAt": row["updated_at"],
}
def catalog_product(row: sqlite3.Row) -> dict:
# Export the complete product, including descriptions, tables, galleries, and detailQueries.
return ensure_product_queries(product_dict(row))
def _known_kiko_families() -> set[str]:
"""Family keys already present locally (from sourceUrl or local slug)."""
known: set[str] = set()
with db() as conn:
rows = conn.execute(
"SELECT content_json, slug FROM products WHERE active = 1"
).fetchall()
for row in rows:
try:
cj = json.loads(row["content_json"] or "{}")
except json.JSONDecodeError:
cj = {}
src = str(cj.get("sourceUrl") or "").strip()
if src and "/p/" in src:
known.add(_family_key_from_product_url(src))
slug = str(row["slug"] or "").strip().lower()
if slug:
# strip trailing shade-sku pattern if present
m = re.match(r"^(.+)-(\d+)-(\d+)$", slug)
known.add(m.group(1) if m else slug)
return known
def _slugify(value: str) -> str:
s = (value or "").strip().lower()
s = re.sub(r"[^\w\s-]", "", s, flags=re.UNICODE)
s = re.sub(r"[-\s]+", "-", s).strip("-")
return s or f"item-{secrets.token_hex(3)}"
def _pick(d: dict, *keys, default=None):
for k in keys:
if k in d and d[k] is not None:
return d[k]
return default
def normalize_variants(raw_variants, *, product_id: str, product_name: str, product_slug: str) -> list:
"""Normalize color/shade variants to the storefront colorVariants shape."""
if raw_variants is None:
return []
if isinstance(raw_variants, str):
raw_variants = json.loads(raw_variants) if raw_variants.strip() else []
if not isinstance(raw_variants, list):
raise ValueError("colorVariants must be a list")
out = []
seen_slugs = set()
seen_ids = set()
for i, raw in enumerate(raw_variants):
if not isinstance(raw, dict):
raise ValueError("Each variant must be an object")
validate_content(raw)
shade = str(_pick(raw, "shade", "shadeCode", "label", "color", default="") or "").strip()
if not shade:
shade = " ".join(str(raw[k]) for k in ("colorCode", "colorName") if raw.get(k))
if not shade and not _pick(raw, "slug", "id", default=None):
continue
vid = str(_pick(raw, "id", default="") or "").strip() or f"{product_id}-v{i+1}"
if vid in seen_ids:
raise ValueError("Duplicate variant id: " + vid)
seen_ids.add(vid)
vslug = str(_pick(raw, "slug", default="") or "").strip()
if not vslug:
base = _slugify(f"{product_slug}-{shade}" if shade else f"{product_slug}-v{i+1}")
vslug = base
else:
vslug = _slugify(vslug)
# keep slugs unique inside one product
original = vslug
n = 2
while vslug in seen_slugs:
vslug = f"{original}-{n}"
n += 1
seen_slugs.add(vslug)
compare = _pick(raw, "compareAtPrice", "compare_at_price", default=None)
compare = float(compare) if compare not in (None, "") else None
images = _pick(raw, "images", "gallery", default=[]) or []
if isinstance(images, str):
images = [x.strip() for x in images.split(",") if x.strip()]
if not isinstance(images, list):
images = []
out.append(
{
**raw,
"id": vid,
"slug": vslug,
"name": str(_pick(raw, "name", default="") or product_name).strip() or product_name,
"shade": shade or str(raw.get("shadeCode") or raw.get("colorName") or raw.get("colorCode") or vslug),
"price": float(_pick(raw, "price", default=0) or 0),
"compareAtPrice": compare,
"stock": int(_pick(raw, "stock", default=0) or 0),
"heroImage": _pick(raw, "heroImage", "hero_image", default=None) or None,
"sku": _pick(raw, "sku", default=None) or None,
"externalId": _pick(raw, "externalId", "external_id", default=None) or None,
"colorHex": _pick(raw, "colorHex", "color_hex", "hex", default=None) or None,
"images": [str(x) for x in images if x],
}
)
return out
def validate_content(raw):
for key in ("price", "compareAtPrice", "stock"):
if raw.get(key) not in (None, ""):
value = float(raw[key])
if not math.isfinite(value) or value < 0 or (key == "stock" and not value.is_integer()):
raise ValueError(key + " must be a finite nonnegative number (integer for stock)")
for key in ("details", "specifications"):
if key in raw and not isinstance(raw[key], (list, dict)):
raise ValueError(key + " must be an array of label/value rows or an object")
if isinstance(raw.get(key), list) and any(not isinstance(r, dict) or not (r.get("label") or r.get("key")) or "value" not in r for r in raw[key]):
raise ValueError(key + " rows require label (or key) and value")
if "detailQueries" in raw and raw["detailQueries"] is not None and not isinstance(raw["detailQueries"], dict):
raise ValueError("detailQueries must be an object of key → search query strings")
if "blocks" in raw and not isinstance(raw["blocks"], list):
raise ValueError("blocks must be an array")
def normalize_product_input(raw: dict, existing_id: str | None = None) -> dict:
validate_content(raw)
# Normalize details + attach scalable SerpAPI/detail queries before persist/export.
ensure_product_queries(raw)
name = str(_pick(raw, "name", default="") or "").strip()
if not name:
raise ValueError("name is required")
slug = str(_pick(raw, "slug", default="") or "").strip() or _slugify(name)
slug = _slugify(slug)
pid = str(_pick(raw, "id", default="") or existing_id or uuid.uuid4().hex)
variants = normalize_variants(
_pick(raw, "colorVariants", "variants", "color_variants", default=[]),
product_id=pid,
product_name=name,
product_slug=slug,
)
price = float(_pick(raw, "price", default=0) or 0)
compare = _pick(raw, "compareAtPrice", "compare_at_price", default=None)
compare = float(compare) if compare not in (None, "") else None
stock = int(_pick(raw, "stock", default=0) or 0)
# Variants drive stock total; fill missing base price/image from primary shade.
hero_image = _pick(raw, "heroImage", "hero_image", default=None)
if variants:
primary = next((v for v in variants if int(v.get("stock") or 0) > 0), variants[0])
stock = sum(int(v.get("stock") or 0) for v in variants)
if price == 0 and primary.get("price"):
price = float(primary["price"] or 0)
if compare is None and primary.get("compareAtPrice") is not None:
compare = primary["compareAtPrice"]
if not hero_image and primary.get("heroImage"):
hero_image = primary["heroImage"]
active = _pick(raw, "active", default=True)
if isinstance(active, str):
active = active.lower() in ("1", "true", "yes", "on")
return {
"id": pid,
"slug": slug,
"name": name,
"name_fa": str(_pick(raw, "nameFa", "name_fa", default="") or ""),
"price": price,
"compare_at_price": compare,
"stock": stock,
"hero_image": hero_image,
"category_slug": _pick(raw, "category", "category_slug", default=None) or None,
"category_fa": (
str(_pick(raw, "categoryFa", "category_fa", default="") or "").strip()
or persian_category_label(
str(_pick(raw, "category", "category_slug", default="") or "")
)
),
"description": str(_pick(raw, "description", default="") or ""),
"sku": _pick(raw, "sku", default=None) or None,
"external_id": _pick(raw, "externalId", "external_id", default=None) or None,
"source": str(_pick(raw, "source", default="manual") or "manual"),
"color_variants": json.dumps(variants, ensure_ascii=False),
"content_json": json.dumps({k: v for k, v in raw.items() if k not in ("variants", "colorVariants", "color_variants")}, ensure_ascii=False),
"active": 1 if active else 0,
}
def upsert_product(conn: sqlite3.Connection, data: dict) -> None:
ts = _now()
conn.execute(
"""
INSERT INTO products (
id, slug, name, name_fa, price, compare_at_price, stock, hero_image,
category_slug, category_fa, description, sku, external_id, source,
color_variants, active, created_at, updated_at
) VALUES (
:id, :slug, :name, :name_fa, :price, :compare_at_price, :stock, :hero_image,
:category_slug, :category_fa, :description, :sku, :external_id, :source,
:color_variants, :active, :created_at, :updated_at
)
ON CONFLICT(id) DO UPDATE SET
slug=excluded.slug,
name=excluded.name,
name_fa=excluded.name_fa,
price=excluded.price,
compare_at_price=excluded.compare_at_price,
stock=excluded.stock,
hero_image=excluded.hero_image,
category_slug=excluded.category_slug,
category_fa=excluded.category_fa,
description=excluded.description,
sku=excluded.sku,
external_id=excluded.external_id,
source=excluded.source,
color_variants=excluded.color_variants,
active=excluded.active,
updated_at=excluded.updated_at
""",
{**data, "created_at": ts, "updated_at": ts},
)
conn.execute("UPDATE products SET content_json = ? WHERE id = ?", (data["content_json"], data["id"]))
def upsert_category(conn: sqlite3.Connection, raw: dict) -> None:
slug = str(raw.get("slug") or "").strip()
if not slug:
raise ValueError("slug is required")
name = str(raw.get("name") or slug).strip()
name_fa = str(raw.get("name_fa") or raw.get("nameFa") or "").strip()
if not name_fa:
seed_fa = seed_name_fa_by_slug().get(slug, "")
name_fa = seed_fa or persian_category_label(name) or persian_category_label(
name.replace("-", " ")
)
parent = raw.get("parent_slug") if "parent_slug" in raw else raw.get("parentSlug")
parent = (str(parent).strip() or None) if parent else None
sort_order = int(raw.get("sort_order", raw.get("sortOrder", 0)) or 0)
active = raw.get("active", True)
if isinstance(active, str):
active = active.lower() in ("1", "true", "yes", "on")
ts = _now()
conn.execute(
"""
INSERT INTO categories (slug, name, name_fa, parent_slug, sort_order, active, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(slug) DO UPDATE SET
name=excluded.name,
name_fa=CASE
WHEN excluded.name_fa != '' THEN excluded.name_fa
ELSE categories.name_fa
END,
parent_slug=excluded.parent_slug,
sort_order=excluded.sort_order,
active=excluded.active,
updated_at=excluded.updated_at
""",
(slug, name, name_fa, parent, sort_order, 1 if active else 0, ts, ts),
)
def find_existing_product(conn: sqlite3.Connection, raw: dict):
"""Resolve an existing family for upsert so KIKO exports do not create duplicates."""
rid = raw.get("id")
if rid:
row = conn.execute("SELECT * FROM products WHERE id = ?", (str(rid),)).fetchone()
if row:
return row
slug = str(raw.get("slug") or "").strip()
if slug:
row = conn.execute("SELECT * FROM products WHERE slug = ?", (slug,)).fetchone()
if row:
return row
sku = str(raw.get("sku") or "").strip()
if sku:
row = conn.execute(
"SELECT * FROM products WHERE sku = ? AND active = 1 ORDER BY length(description) DESC LIMIT 1",
(sku,),
).fetchone()
if row:
return row
external = str(raw.get("externalId") or raw.get("external_id") or "").strip()
if external:
row = conn.execute(
"SELECT * FROM products WHERE external_id = ? AND active = 1 LIMIT 1",
(external,),
).fetchone()
if row:
return row
match = raw.get("match") if isinstance(raw.get("match"), dict) else {}
for key in match.get("slugs") or []:
key = str(key or "").strip()
if not key:
continue
row = conn.execute("SELECT * FROM products WHERE slug = ?", (key,)).fetchone()
if row:
return row
name = str(raw.get("name") or match.get("name") or "").strip()
if name:
row = conn.execute(
"SELECT * FROM products WHERE active = 1 AND lower(name) = ? LIMIT 1",
(name.lower(),),
).fetchone()
if row:
return row
# "Love Fusion … Foundation" should match "Love Fusion … Foundation 0NR"
row = conn.execute(
"""
SELECT * FROM products
WHERE active = 1 AND lower(name) LIKE ?
ORDER BY length(description) DESC, length(color_variants) DESC
LIMIT 1
""",
(name.lower() + " %",),
).fetchone()
if row:
return row
return None
def import_products_payload(payload) -> dict:
if isinstance(payload, dict):
products = payload.get("products")
if products is None:
raise ValueError("JSON must contain a products array")
elif isinstance(payload, list):
products = payload
else:
raise ValueError("invalid JSON")
if not isinstance(products, list):
raise ValueError("products must be an array")
created = updated = skipped = 0
errors = []
with db() as conn:
if isinstance(payload, dict):
for key in ("metadata", "images"):
if key in payload:
value = payload[key]
if key == "metadata":
if not isinstance(value, dict):
raise ValueError("metadata must be an object")
prior = conn.execute("SELECT value FROM catalog_metadata WHERE key='metadata'").fetchone()
value = {**(json.loads(prior[0]) if prior else {}), **value}
if key == "images":
if not isinstance(value, list):
raise ValueError("images must be an array")
prior = conn.execute("SELECT value FROM catalog_metadata WHERE key='images'").fetchone()
merged = {x.get("original_url") or x.get("id"): x for x in (json.loads(prior[0]) if prior else [])}
for x in value:
if not isinstance(x, dict) or not (x.get("original_url") or x.get("id")):
raise ValueError("image mapping requires original_url or id")
merged[x.get("original_url") or x.get("id")] = x
value = list(merged.values())
conn.execute("INSERT INTO catalog_metadata VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, json.dumps(value, ensure_ascii=False)))
for i, raw in enumerate(products):
if not isinstance(raw, dict):
skipped += 1
continue
try:
existing = find_existing_product(conn, raw)
if existing:
incoming = dict(raw)
if "variants" in incoming and "colorVariants" not in incoming:
incoming["colorVariants"] = incoming.pop("variants")
if "color_variants" in incoming and "colorVariants" not in incoming:
incoming["colorVariants"] = incoming.pop("color_variants")
# Keep local Persian/SEO text when incoming description is English source fill.
prior = product_dict(existing)
for key in ("description", "seoTitle", "seoDescription", "nameFa"):
incoming_val = incoming.get(key)
prior_val = prior.get(key)
if (prior_val or "").strip() and (
not (incoming_val or "").strip()
or incoming.get("source") == "kikocosmetics"
):
# Do not overwrite reviewed local copy with KIKO English on upsert.
if key == "description" and incoming.get("source") == "kikocosmetics":
incoming["sourceDescription"] = incoming.get("sourceDescription") or incoming_val
incoming[key] = prior_val
elif (prior_val or "").strip():
incoming[key] = prior_val
# Keep existing storefront slug/id so we never create a second family page.
raw = {
**prior,
**incoming,
"id": existing["id"],
"slug": prior.get("slug") or incoming.get("slug"),
}
data = normalize_product_input(raw, existing["id"] if existing else None)
# Prefer slug uniqueness when colliding with different id
clash = conn.execute(
"SELECT id FROM products WHERE slug = ? AND id != ?",
(data["slug"], data["id"]),
).fetchone()
if clash:
data["slug"] = f"{data['slug']}-{data['id'][:6]}"
upsert_product(conn, data)
if existing:
updated += 1
else:
created += 1
except Exception as exc:
errors.append({"index": i, "error": str(exc)})
skipped += 1
return {"created": created, "updated": updated, "skipped": skipped, "errors": errors[:50]}
def apply_stock_updates(updates: list) -> dict:
matched = updated = 0
with db() as conn:
for u in updates:
if not isinstance(u, dict) or u.get("stock") is None:
continue
stock = int(u["stock"])
row = None
for key, col in (
("id", "id"),
("external_id", "external_id"),
("externalId", "external_id"),
("sku", "sku"),
("slug", "slug"),
):
val = u.get(key)
if val:
row = conn.execute(
f"SELECT id FROM products WHERE {col} = ?", (str(val),)
).fetchone()
if row:
break
if not row:
continue
matched += 1
conn.execute(
"UPDATE products SET stock = ?, updated_at = ? WHERE id = ?",
(stock, _now(), row["id"]),
)
updated += 1
return {"matched": matched, "updated": updated}
def export_catalog() -> dict:
with db() as conn:
rows = conn.execute(
"SELECT * FROM products WHERE active = 1 ORDER BY updated_at DESC"
).fetchall()
categories = [category_dict(r) for r in conn.execute("SELECT * FROM categories ORDER BY id")]
extra = {r["key"]: json.loads(r["value"]) for r in conn.execute("SELECT * FROM catalog_metadata")}
# Shared detail-query templates travel with the catalog for future imports/tools.
templates = extra.get("detailQueryTemplates") or default_catalog_detail_queries()
extra["detailQueryTemplates"] = templates
with db() as conn:
conn.execute(
"INSERT INTO catalog_metadata VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
("detailQueryTemplates", json.dumps(templates, ensure_ascii=False)),
)
payload = {
**extra,
"version": "dedupe2",
"schemaVersion": 2,
"categories": categories,
"products": [catalog_product(r) for r in rows],
"exportedAt": _now(),
}
DIST_CATALOG.parent.mkdir(parents=True, exist_ok=True)
atomic_json(DIST_CATALOG, payload)
return {"path": str(DIST_CATALOG), "count": len(payload["products"])}
def atomic_json(destination: Path, payload) -> None:
destination.parent.mkdir(parents=True, exist_ok=True)
fd, name = tempfile.mkstemp(dir=destination.parent, prefix=".catalog-", suffix=".json")
try:
with os.fdopen(fd, "w", encoding="utf-8") as stream:
json.dump(payload, stream, ensure_ascii=False)
stream.flush()
os.fsync(stream.fileno())
os.chmod(name, 0o644)
os.replace(name, destination)
finally:
if os.path.exists(name):
os.unlink(name)
def export_import_bundle(destination: Path) -> dict:
"""Private lossless interchange export; includes inactive products, no credentials."""
payload = import_bundle()
atomic_json(destination, payload)
return {"path": str(destination), "count": len(payload["products"])}
def import_bundle() -> dict:
with db() as conn:
payload = {r["key"]: json.loads(r["value"]) for r in conn.execute("SELECT * FROM catalog_metadata")}
payload.update(schemaVersion=2, products=[product_dict(r) for r in conn.execute("SELECT * FROM products ORDER BY id")],
categories=[category_dict(r) for r in conn.execute("SELECT * FROM categories ORDER BY id")])
return payload
def checkpoint_database() -> None:
"""Flush WAL into the main .db file so downloads are self-contained."""
if not DB_PATH.is_file():
return
with db() as conn:
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
def validate_sqlite_backup(path: Path) -> dict:
raw = path.read_bytes()[:16]
if not raw.startswith(SQLITE_HEADER):
raise ValueError("file is not a SQLite database")
conn = sqlite3.connect(str(path))
try:
tables = {
r[0]
for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
}
required = {"products", "categories", "users"}
missing = required - tables
if missing:
raise ValueError(
"missing required tables: " + ", ".join(sorted(missing))
)
products = conn.execute("SELECT COUNT(*) FROM products").fetchone()[0]
categories = conn.execute("SELECT COUNT(*) FROM categories").fetchone()[0]
users = conn.execute("SELECT COUNT(*) FROM users").fetchone()[0]
return {
"products": products,
"categories": categories,
"users": users,
"bytes": path.stat().st_size,
}
finally:
conn.close()
def restore_database_bytes(data: bytes) -> dict:
if len(data) < 100:
raise ValueError("file too small")
if not data.startswith(SQLITE_HEADER):
raise ValueError("file is not a SQLite database")
DATA_DIR.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(
prefix="kikomiko-restore-", suffix=".db", dir=DATA_DIR, delete=False
) as tmp:
tmp.write(data)
tmp_path = Path(tmp.name)
try:
stats = validate_sqlite_backup(tmp_path)
checkpoint_database()
bak = DATA_DIR / f"kikomiko.db.bak-{_now()}"
if DB_PATH.is_file():
shutil.copy2(DB_PATH, bak)
# Drop WAL sidecars so the restored main file is authoritative.
for side in (DATA_DIR / "kikomiko.db-wal", DATA_DIR / "kikomiko.db-shm"):
if side.is_file():
side.unlink()
os.replace(tmp_path, DB_PATH)
init_db()
stats["backup"] = str(bak) if bak.is_file() else None
return stats
except Exception:
if tmp_path.is_file():
tmp_path.unlink()
raise
# --- HTTP handler ---------------------------------------------------------------
class AdminHandler(BaseHTTPRequestHandler):
server_version = "KikomikoAdmin/1.0"
def log_message(self, fmt, *args):
sys.stderr.write("%s - %s\n" % (self.address_string(), fmt % args))
def _json(self, status: int, payload: dict):
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.send_header("X-Content-Type-Options", "nosniff")
self.end_headers()
self.wfile.write(body)
def _text(self, status: int, text: str, content_type="text/plain; charset=utf-8"):
body = text.encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Content-Type-Options", "nosniff")
self.end_headers()
self.wfile.write(body)
def _read_json(self) -> dict:
length = int(self.headers.get("Content-Length") or 0)
if length > 8_000_000:
raise ValueError("payload too large")
raw = self.rfile.read(length) if length else b"{}"
if not raw:
return {}
data = json.loads(raw.decode("utf-8"))
if not isinstance(data, dict):
raise ValueError("JSON object required")
return data
def _cookies(self) -> SimpleCookie:
c = SimpleCookie()
if "Cookie" in self.headers:
c.load(self.headers["Cookie"])
return c
def _client_ip(self) -> str:
return self.client_address[0] or "unknown"
def _get_session(self):
token = None
cookies = self._cookies()
if "session" in cookies:
token = cookies["session"].value
if not token:
return None
with db() as conn:
row = conn.execute(
"""
SELECT s.token, s.csrf, s.expires_at, u.id AS user_id, u.username
FROM sessions s JOIN users u ON u.id = s.user_id
WHERE s.token = ?
""",
(token,),
).fetchone()
if not row or row["expires_at"] < _now():
if row:
conn.execute("DELETE FROM sessions WHERE token = ?", (token,))
return None
# sliding expiry
conn.execute(
"UPDATE sessions SET expires_at = ? WHERE token = ?",
(_now() + SESSION_TTL, token),
)
return dict(row)
def _require_session(self):
sess = self._get_session()
if not sess:
self._json(HTTPStatus.UNAUTHORIZED, {"error": "unauthorized"})
return None
return sess
def _require_csrf(self, sess) -> bool:
header = self.headers.get("X-CSRF-Token") or ""
if not header or not hmac.compare_digest(header, sess["csrf"]):
self._json(HTTPStatus.FORBIDDEN, {"error": "invalid csrf token"})
return False
return True
def _set_session_cookie(self, token: str, max_age: int):
flags = f"session={token}; Path=/; HttpOnly; SameSite=Strict; Max-Age={max_age}"
self.send_header("Set-Cookie", flags)
def _clear_session_cookie(self):
self.send_header(
"Set-Cookie",
"session=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0",
)
def _login_allowed(self, ip: str) -> bool:
with db() as conn:
row = conn.execute(
"SELECT fails, window_start FROM login_attempts WHERE ip = ?", (ip,)
).fetchone()
if not row:
return True
if _now() - row["window_start"] > LOGIN_WINDOW:
conn.execute("DELETE FROM login_attempts WHERE ip = ?", (ip,))
return True
return row["fails"] < LOGIN_MAX_FAILS
def _record_login_fail(self, ip: str):
with db() as conn:
row = conn.execute(
"SELECT fails, window_start FROM login_attempts WHERE ip = ?", (ip,)
).fetchone()
if not row or _now() - row["window_start"] > LOGIN_WINDOW:
conn.execute(
"INSERT OR REPLACE INTO login_attempts (ip, fails, window_start) VALUES (?, 1, ?)",
(ip, _now()),
)
else:
conn.execute(
"UPDATE login_attempts SET fails = fails + 1 WHERE ip = ?", (ip,)
)
def _clear_login_fails(self, ip: str):
with db() as conn:
conn.execute("DELETE FROM login_attempts WHERE ip = ?", (ip,))
# --- routing ----------------------------------------------------------------
def do_GET(self):
parsed = urlparse(self.path)
path = parsed.path
if path.startswith("/api/"):
return self._api_get(path, parsed)
return self._serve_static(path)
def do_POST(self):
parsed = urlparse(self.path)
path = parsed.path
if path.startswith("/api/"):
return self._api_post(path)
self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
def do_PUT(self):
parsed = urlparse(self.path)
if parsed.path.startswith("/api/"):
return self._api_put(parsed.path)
self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
def do_DELETE(self):
parsed = urlparse(self.path)
if parsed.path.startswith("/api/"):
return self._api_delete(parsed.path)
self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
def _serve_static(self, path: str):
if path in ("", "/"):
path = "/index.html"
rel = path.lstrip("/")
parts = Path(rel).parts
if not parts or ".." in parts or parts[0].startswith("/"):
return self._text(HTTPStatus.FORBIDDEN, "forbidden")
# Join without resolving first so symlink hops (local fonts) stay allowed.
file_path = STATIC_DIR.joinpath(*parts)
if not file_path.is_file():
return self._text(HTTPStatus.NOT_FOUND, "not found")
# Final path must stay inside the kikomiko project tree.
resolved = file_path.resolve()
project_root = ROOT.parent.resolve()
try:
resolved.relative_to(project_root)
except ValueError:
return self._text(HTTPStatus.FORBIDDEN, "forbidden")
ext = resolved.suffix.lower()
types = {
".html": "text/html; charset=utf-8",
".css": "text/css; charset=utf-8",
".js": "application/javascript; charset=utf-8",
".woff": "font/woff",
".woff2": "font/woff2",
".png": "image/png",
".svg": "image/svg+xml",
".json": "application/json; charset=utf-8",
}
data = resolved.read_bytes()
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", types.get(ext, "application/octet-stream"))
self.send_header("Content-Length", str(len(data)))
self.send_header("X-Content-Type-Options", "nosniff")
if ext in (".html", ".js", ".css"):
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(data)
# --- API GET ----------------------------------------------------------------
def _api_get(self, path: str, parsed):
if path == "/api/health":
return self._json(HTTPStatus.OK, {"ok": True})
if path == "/api/session":
sess = self._get_session()
if not sess:
return self._json(HTTPStatus.OK, {"authenticated": False})
return self._json(
HTTPStatus.OK,
{
"authenticated": True,
"username": sess["username"],
"csrf": sess["csrf"],
},
)
sess = self._require_session()
if not sess:
return
if path == "/api/export/json":
return self._json(HTTPStatus.OK, import_bundle())
if path == "/api/categories":
with db() as conn:
rows = conn.execute(
"SELECT * FROM categories ORDER BY sort_order, name_fa, name"
).fetchall()
return self._json(HTTPStatus.OK, {"categories": [category_dict(r) for r in rows]})
if path == "/api/products":
q = parse_qs(parsed.query)
search = (q.get("q") or [""])[0].strip().lower()
with db() as conn:
rows = conn.execute(
"SELECT * FROM products ORDER BY updated_at DESC LIMIT 2000"
).fetchall()
products = [product_dict(r) for r in rows]
if search:
products = [
p
for p in products
if search
in f"{p['name']} {p['slug']} {p['sku'] or ''} {p['category'] or ''}".lower()
]
return self._json(HTTPStatus.OK, {"products": products})
if path.startswith("/api/products/"):
pid = path.split("/", 3)[-1]
with db() as conn:
row = conn.execute("SELECT * FROM products WHERE id = ?", (pid,)).fetchone()
if not row:
return self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
return self._json(HTTPStatus.OK, {"product": product_dict(row)})
if path == "/api/fetchers":
return self._json(HTTPStatus.OK, {"fetchers": list_fetchers()})
if path == "/api/stats":
with db() as conn:
pc = conn.execute("SELECT COUNT(*) AS c FROM products").fetchone()["c"]
cc = conn.execute("SELECT COUNT(*) AS c FROM categories").fetchone()["c"]
stock = conn.execute(
"SELECT COALESCE(SUM(stock),0) AS s FROM products WHERE active=1"
).fetchone()["s"]
return self._json(
HTTPStatus.OK,
{"products": pc, "categories": cc, "stockUnits": stock},
)
if path == "/api/settings/ai":
return self._json(HTTPStatus.OK, public_ai_settings())
if path == "/api/export/database":
if not DB_PATH.is_file():
return self._json(HTTPStatus.NOT_FOUND, {"error": "database not found"})
try:
checkpoint_database()
data = DB_PATH.read_bytes()
except Exception as exc:
return self._json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": str(exc)})
stamp = time.strftime("%Y%m%d-%H%M%S")
filename = f"kikomiko-{stamp}.db"
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "application/octet-stream")
self.send_header(
"Content-Disposition", f'attachment; filename="{filename}"'
)
self.send_header("Content-Length", str(len(data)))
self.send_header("Cache-Control", "no-store")
self.send_header("X-Content-Type-Options", "nosniff")
self.end_headers()
self.wfile.write(data)
return
self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
# --- API POST ---------------------------------------------------------------
def _api_post(self, path: str):
if path == "/api/login":
return self._handle_login()
sess = self._require_session()
if not sess:
return
if not self._require_csrf(sess):
return
if path == "/api/import/database":
length = int(self.headers.get("Content-Length") or 0)
if length <= 0:
return self._json(HTTPStatus.BAD_REQUEST, {"error": "empty body"})
if length > DB_UPLOAD_MAX:
return self._json(
HTTPStatus.REQUEST_ENTITY_TOO_LARGE,
{"error": f"file too large (max {DB_UPLOAD_MAX // 1_000_000}MB)"},
)
try:
raw = self.rfile.read(length)
result = restore_database_bytes(raw)
return self._json(HTTPStatus.OK, {"ok": True, **result})
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
try:
body = self._read_json()
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/logout":
with db() as conn:
conn.execute("DELETE FROM sessions WHERE token = ?", (sess["token"],))
self.send_response(HTTPStatus.OK)
self._clear_session_cookie()
body_out = json.dumps({"ok": True}).encode("utf-8")
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body_out)))
self.end_headers()
self.wfile.write(body_out)
return
if path == "/api/categories":
try:
with db() as conn:
upsert_category(conn, body)
row = conn.execute(
"SELECT * FROM categories WHERE slug = ?",
(body.get("slug"),),
).fetchone()
return self._json(HTTPStatus.CREATED, {"category": category_dict(row)})
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/products":
try:
data = normalize_product_input(body)
with db() as conn:
clash = conn.execute(
"SELECT id FROM products WHERE slug = ?", (data["slug"],)
).fetchone()
if clash:
return self._json(
HTTPStatus.CONFLICT, {"error": "slug already exists"}
)
upsert_product(conn, data)
row = conn.execute(
"SELECT * FROM products WHERE id = ?", (data["id"],)
).fetchone()
return self._json(HTTPStatus.CREATED, {"product": product_dict(row)})
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/enrich/kiko":
# Preview enrichment for the product editor (does not save).
try:
fetched = fetch_kiko_product(
url=body.get("url"),
query=body.get("query") or body.get("name"),
market=str(body.get("market") or "en-gb"),
)
return self._json(HTTPStatus.OK, {"product": fetched})
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path.startswith("/api/products/") and path.endswith("/enrich-kiko"):
pid = path[len("/api/products/") : -len("/enrich-kiko")].strip("/")
if not pid or "/" in pid:
return self._json(HTTPStatus.BAD_REQUEST, {"error": "invalid product id"})
try:
with db() as conn:
exists = conn.execute(
"SELECT * FROM products WHERE id = ?", (pid,)
).fetchone()
if not exists:
return self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
existing = product_dict(exists)
fetched = fetch_kiko_product(
url=body.get("url") or existing.get("sourceUrl"),
query=body.get("query")
or existing.get("displayName")
or existing.get("name"),
market=str(body.get("market") or existing.get("sourceMarket") or "en-gb"),
)
merged = merge_enrichment(
existing,
fetched,
overwrite_description=bool(body.get("overwriteDescription")),
)
data = normalize_product_input(merged, existing_id=pid)
data["id"] = pid
upsert_product(conn, data)
row = conn.execute(
"SELECT * FROM products WHERE id = ?", (pid,)
).fetchone()
return self._json(
HTTPStatus.OK,
{
"product": product_dict(row),
"sourceUrl": fetched.get("sourceUrl"),
"detailsCount": len(fetched.get("details") or []),
"variantCount": len(fetched.get("colorVariants") or []),
},
)
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/import/categories":
if not SEED_CATEGORIES.is_file():
return self._json(HTTPStatus.NOT_FOUND, {"error": "seed file missing"})
seed = json.loads(SEED_CATEGORIES.read_text(encoding="utf-8"))
count = 0
with db() as conn:
for item in seed:
upsert_category(conn, item)
count += 1
return self._json(HTTPStatus.OK, {"imported": count})
if path == "/api/import/catalog":
source = body.get("source", "dist")
if source == "dist":
if not DIST_CATALOG.is_file():
return self._json(
HTTPStatus.NOT_FOUND,
{"error": f"catalog not found: {DIST_CATALOG}"},
)
payload = json.loads(DIST_CATALOG.read_text(encoding="utf-8"))
elif source == "inline":
payload = body.get("data")
if payload is None:
return self._json(HTTPStatus.BAD_REQUEST, {"error": "data required"})
elif source == "path":
p = Path(body.get("path") or "").expanduser()
if not p.is_file():
return self._json(HTTPStatus.BAD_REQUEST, {"error": "path not found"})
# Large catalog dumps (100MB+) — path import avoids the HTTP body limit.
payload = json.loads(p.read_text(encoding="utf-8"))
else:
return self._json(HTTPStatus.BAD_REQUEST, {"error": "unknown source"})
cat_count = 0
if isinstance(payload, dict) and isinstance(payload.get("categories"), list):
# Prefer parents before children
cats = sorted(
payload["categories"],
key=lambda c: str((c or {}).get("full_path") or (c or {}).get("id") or "").count("/"),
)
with db() as conn:
for raw in cats:
if not isinstance(raw, dict):
continue
name = str(raw.get("name") or raw.get("full_path") or "").strip()
if not name:
continue
parent_id = raw.get("parent_id") or raw.get("parentSlug")
parent_slug = None
if parent_id:
parent_slug = _slugify(str(parent_id).split(" / ")[-1].strip())
upsert_category(
conn,
{
"slug": raw.get("slug") or _slugify(name),
"name": name.title() if name == name.upper() else name,
"name_fa": raw.get("name_fa") or raw.get("nameFa") or "",
"parent_slug": parent_slug,
"sort_order": raw.get("sort_order") or raw.get("sortOrder") or 0,
"active": raw.get("active", True),
},
)
cat_count += 1
result = import_products_payload(payload)
result["categoriesImported"] = cat_count
return self._json(HTTPStatus.OK, result)
if path == "/api/export/catalog":
try:
result = export_catalog()
return self._json(HTTPStatus.OK, result)
except Exception as exc:
return self._json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": str(exc)})
if path == "/api/fetch/run":
fetcher_id = body.get("fetcher")
action = body.get("action", "fetch")
config = body.get("config") or {}
try:
module = get_fetcher(fetcher_id)
if action == "fetch":
rows = module.fetch_products(config)
result = import_products_payload(rows)
result["fetched"] = len(rows)
return self._json(HTTPStatus.OK, result)
if action == "sync":
with db() as conn:
products = [product_dict(r) for r in conn.execute("SELECT * FROM products")]
updates = module.sync_stocks(config, products)
result = apply_stock_updates(updates)
result["received"] = len(updates)
return self._json(HTTPStatus.OK, result)
return self._json(HTTPStatus.BAD_REQUEST, {"error": "action must be fetch or sync"})
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/enrich/kiko-discover":
# List product family URLs from the official KIKO sitemap.
try:
market = str(body.get("market") or "en-gb").strip() or "en-gb"
one_per_family = body.get("onePerFamily", True)
discovered = discover_catalog_product_urls(
market, one_per_family=bool(one_per_family)
)
skip_existing = body.get("skipExisting", True)
urls = list(discovered.get("urls") or [])
already = 0
if skip_existing:
known_families = _known_kiko_families()
filtered = []
for u in urls:
if _family_key_from_product_url(u) in known_families:
already += 1
continue
filtered.append(u)
urls = filtered
return self._json(
HTTPStatus.OK,
{
"ok": True,
"market": discovered.get("market"),
"sitemapUrl": discovered.get("sitemapUrl"),
"shadeUrls": discovered.get("shadeUrls"),
"familyUrls": discovered.get("familyUrls"),
"alreadyPresent": already,
"pending": len(urls),
"urls": urls,
"hint": "این لیست را می‌توانید وارد کنید یا دکمه ورود همه را بزنید.",
},
)
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/enrich/kiko-bulk":
# Bulk pull from kikocosmetics.com (many URLs, refill existing, or full catalog).
try:
market = str(body.get("market") or "en-gb").strip() or "en-gb"
overwrite_description = bool(body.get("overwriteDescription"))
limit = int(body.get("limit") or 30)
limit = max(1, min(limit, 80))
mode = str(body.get("mode") or "urls").strip() or "urls"
offset = max(0, int(body.get("offset") or 0))
jobs: list[dict] = []
catalog_meta: dict = {}
if mode == "urls":
urls = body.get("urls")
if isinstance(urls, str):
urls = [u.strip() for u in urls.replace(",", "\n").splitlines() if u.strip()]
elif not isinstance(urls, list):
urls = []
urls = [str(u).strip() for u in urls if str(u).strip()]
# de-dupe, keep order
seen = set()
uniq = []
for u in urls:
if u in seen:
continue
seen.add(u)
uniq.append(u)
for u in uniq[:limit]:
jobs.append({"url": u})
catalog_meta = {"pendingAfter": max(0, len(uniq) - limit)}
elif mode == "existing":
only_missing = body.get("onlyMissing", True)
with db() as conn:
rows = conn.execute(
"SELECT * FROM products WHERE active = 1 ORDER BY updated_at DESC"
).fetchall()
for row in rows:
p = product_dict(row)
has_details = bool(p.get("details"))
has_desc = bool((p.get("description") or "").strip())
if only_missing and has_details and has_desc:
continue
if not (p.get("sourceUrl") or p.get("name") or p.get("sku")):
continue
jobs.append(
{
"id": p.get("id"),
"url": p.get("sourceUrl") or "",
"query": p.get("name") or "",
"existing": p,
}
)
if len(jobs) >= limit:
break
elif mode == "catalog":
# Discover from sitemap, skip known products, take next batch.
skip_existing = body.get("skipExisting", True)
discovered = discover_catalog_product_urls(market, one_per_family=True)
urls = list(discovered.get("urls") or [])
already = 0
if skip_existing:
known_families = _known_kiko_families()
filtered = []
for u in urls:
if _family_key_from_product_url(u) in known_families:
already += 1
continue
filtered.append(u)
urls = filtered
already = max(0, int(discovered.get("familyUrls") or 0) - len(urls))
batch = urls[offset : offset + limit]
for u in batch:
jobs.append({"url": u})
catalog_meta = {
"shadeUrls": discovered.get("shadeUrls"),
"familyUrls": discovered.get("familyUrls"),
"alreadyPresent": already,
"pendingTotal": len(urls),
"offset": offset,
"pendingAfter": max(0, len(urls) - (offset + len(batch))),
"done": len(urls) <= (offset + len(batch)),
}
else:
return self._json(
HTTPStatus.BAD_REQUEST,
{"error": "mode must be urls, existing, or catalog"},
)
if not jobs:
return self._json(
HTTPStatus.OK,
{
"ok": True,
"mode": mode,
"fetched": 0,
"created": 0,
"updated": 0,
"errors": [],
"done": True,
"pendingAfter": 0,
"hint": "چیزی برای دریافت نمانده.",
**catalog_meta,
},
)
fetched_rows = []
errors = []
for i, job in enumerate(jobs):
try:
fetched = fetch_kiko_product(
url=job.get("url") or None,
query=(job.get("query") or None) if not job.get("url") else None,
market=market,
)
existing = job.get("existing")
if existing:
fetched_rows.append(
merge_enrichment(
existing,
fetched,
overwrite_description=overwrite_description,
)
)
else:
fetched_rows.append(fetched)
except Exception as exc: # noqa: BLE001
errors.append(
{
"url": job.get("url") or "",
"query": job.get("query") or "",
"error": str(exc),
}
)
if mode == "catalog" and i + 1 < len(jobs):
time.sleep(0.35)
result = import_products_payload(fetched_rows) if fetched_rows else {
"created": 0,
"updated": 0,
"skipped": 0,
"errors": [],
}
result.update(
{
"ok": True,
"mode": mode,
"fetched": len(fetched_rows),
"jobs": len(jobs),
"fetchErrors": errors[:40],
"hint": "بعد از اتمام، از داشبورد «خروجی به catalog.json» را بزنید تا روی سایت بیاید.",
**catalog_meta,
}
)
if "done" not in result:
result["done"] = catalog_meta.get("pendingAfter", 0) == 0
return self._json(HTTPStatus.OK, result)
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/password":
current = body.get("current") or ""
new_pass = body.get("newPassword") or ""
if len(new_pass) < 10:
return self._json(
HTTPStatus.BAD_REQUEST,
{"error": "new password must be at least 10 characters"},
)
with db() as conn:
user = conn.execute(
"SELECT * FROM users WHERE id = ?", (sess["user_id"],)
).fetchone()
if not user or not _verify_password(current, user["password_hash"]):
return self._json(HTTPStatus.FORBIDDEN, {"error": "current password wrong"})
conn.execute(
"UPDATE users SET password_hash = ? WHERE id = ?",
(_hash_password(new_pass), user["id"]),
)
conn.execute("DELETE FROM sessions WHERE user_id = ?", (user["id"],))
return self._json(HTTPStatus.OK, {"ok": True, "relogin": True})
if path == "/api/settings/ai":
try:
kwargs = {}
if "apiKey" in body:
kwargs["api_key"] = str(body.get("apiKey") or "")
if "baseUrl" in body:
kwargs["base_url"] = str(body.get("baseUrl") or "")
if "model" in body:
kwargs["model"] = str(body.get("model") or "")
if "serpApiKey" in body:
kwargs["serp_api_key"] = str(body.get("serpApiKey") or "")
if "categoryApiKey" in body:
kwargs["category_api_key"] = str(body.get("categoryApiKey") or "")
result = save_ai_settings(**kwargs) if kwargs else public_ai_settings()
return self._json(HTTPStatus.OK, {"ok": True, **result})
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/translate/categories":
try:
# Local dictionary works without an API key; AI is only a fallback.
settings = load_ai_settings()
overwrite = bool(body.get("overwrite"))
dry_run = bool(body.get("dryRun"))
limit = int(body.get("limit") or 50)
limit = max(1, min(limit, 200))
with db() as conn:
rows = conn.execute(
"SELECT * FROM categories ORDER BY sort_order, name"
).fetchall()
candidates = []
for r in rows:
cat = category_dict(r)
if category_needs_fa(cat, overwrite=overwrite):
candidates.append(cat)
remaining = max(0, len(candidates) - limit)
batch = candidates[:limit]
results = []
translated = 0
errors = []
for cat in batch:
if dry_run:
results.append(
{
"slug": cat.get("slug"),
"name": cat.get("name"),
"nameFa": cat.get("nameFa") or "",
}
)
continue
try:
stats = translate_category_name(
cat, overwrite=overwrite, settings=settings
)
results.append(stats)
if stats.get("changed"):
# Force-write Persian label (bypass empty-preserve CASE).
conn.execute(
"""
UPDATE categories
SET name_fa = ?, updated_at = ?
WHERE slug = ?
""",
(cat.get("nameFa") or "", _now(), cat["slug"]),
)
translated += 1
except Exception as exc: # noqa: BLE001
errors.append(f"{cat.get('slug')}: {exc}")
results.append(
{
"slug": cat.get("slug"),
"changed": False,
"errors": [str(exc)],
}
)
if any(
code in str(exc)
for code in ("HTTP 401", "HTTP 403", "HTTP 429")
):
break
return self._json(
HTTPStatus.OK,
{
"ok": True,
"dryRun": dry_run,
"processed": len(results),
"translatedCategories": translated,
"remaining": remaining,
"hasAiKey": bool(resolve_category_api_key(settings)),
"errors": errors[:20],
"results": results[:80],
"hint": "Export catalog after translating so the storefront mega-menu updates.",
},
)
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/enrich/details-serp":
try:
settings = load_ai_settings()
serp_key = (settings.get("serpApiKey") or "").strip()
if not serp_key:
return self._json(
HTTPStatus.BAD_REQUEST,
{"error": "SerpAPI key is missing. Save it in Settings first."},
)
overwrite = bool(body.get("overwrite"))
dry_run = bool(body.get("dryRun"))
limit = int(body.get("limit") or 15)
limit = max(1, min(limit, 50))
only_active = body.get("onlyActive", True)
with db() as conn:
if only_active:
rows = conn.execute(
"SELECT * FROM products WHERE active = 1 ORDER BY updated_at DESC"
).fetchall()
else:
rows = conn.execute(
"SELECT * FROM products ORDER BY updated_at DESC"
).fetchall()
candidates = []
for row in rows:
product = ensure_product_queries(product_dict(row))
missing = missing_detail_keys(product, overwrite=overwrite)
if missing:
candidates.append((product, missing))
remaining = max(0, len(candidates) - limit)
batch = candidates[:limit]
results = []
filled_products = 0
filled_fields = 0
errors = []
for product, missing in batch:
if dry_run:
results.append(
{
"id": product.get("id"),
"slug": product.get("slug"),
"missing": missing,
"queries": {
k: (product.get("detailQueries") or {}).get(k)
for k in missing
},
}
)
continue
stats = fill_details_via_serp(
product,
api_key=serp_key,
keys=missing,
overwrite=overwrite,
)
results.append(stats)
if stats.get("errors"):
errors.extend(
f"{product.get('slug')}: {e}" for e in stats["errors"]
)
if any(
"HTTP 401" in e or "HTTP 403" in e or "HTTP 429" in e
for e in stats["errors"]
):
break
if stats.get("filled"):
data = normalize_product_input(
product, existing_id=product["id"]
)
data["id"] = product["id"]
upsert_product(conn, data)
filled_products += 1
filled_fields += len(stats["filled"])
return self._json(
HTTPStatus.OK,
{
"ok": True,
"dryRun": dry_run,
"processed": len(results),
"filledProducts": filled_products,
"filledFields": filled_fields,
"remaining": remaining,
"errors": errors[:20],
"results": results[:40],
"hint": "Each product stores detailQueries in the catalog export for future imports.",
},
)
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path == "/api/translate/descriptions":
try:
settings = load_ai_settings()
if not (settings.get("apiKey") or "").strip():
return self._json(
HTTPStatus.BAD_REQUEST,
{"error": "AI API key is missing. Save it in Settings first."},
)
overwrite = bool(body.get("overwrite"))
dry_run = bool(body.get("dryRun"))
limit = int(body.get("limit") or 25)
limit = max(1, min(limit, 100))
only_active = body.get("onlyActive", True)
with db() as conn:
if only_active:
rows = conn.execute(
"SELECT * FROM products WHERE active = 1 ORDER BY updated_at DESC"
).fetchall()
else:
rows = conn.execute(
"SELECT * FROM products ORDER BY updated_at DESC"
).fetchall()
candidates = []
for row in rows:
product = product_dict(row)
jobs = collect_jobs(product, overwrite=overwrite)
if jobs:
candidates.append(product)
remaining = max(0, len(candidates) - limit)
batch = candidates[:limit]
results = []
translated_products = 0
translated_fields = 0
errors = []
for product in batch:
if dry_run:
jobs = collect_jobs(product, overwrite=overwrite)
results.append(
{
"id": product.get("id"),
"slug": product.get("slug"),
"pending": [j[0] for j in jobs],
}
)
continue
stats = translate_product(
product, overwrite=overwrite, settings=settings
)
results.append(stats)
if stats.get("errors"):
errors.extend(
f"{product.get('slug')}: {e}" for e in stats["errors"]
)
# Stop the whole batch on auth/quota style failures
if any(
"HTTP 401" in e or "HTTP 403" in e or "HTTP 429" in e
for e in stats["errors"]
):
break
if stats.get("changed"):
data = normalize_product_input(product, existing_id=product["id"])
data["id"] = product["id"]
upsert_product(conn, data)
translated_products += 1
translated_fields += len(stats["changed"])
return self._json(
HTTPStatus.OK,
{
"ok": True,
"dryRun": dry_run,
"processed": len(results),
"translatedProducts": translated_products,
"translatedFields": translated_fields,
"remaining": remaining,
"errors": errors[:20],
"results": results[:50],
"hint": "Export catalog to storefront after translating.",
},
)
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
def _handle_login(self):
ip = self._client_ip()
if not self._login_allowed(ip):
return self._json(
HTTPStatus.TOO_MANY_REQUESTS,
{"error": "too many failed attempts; try later"},
)
try:
body = self._read_json()
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
username = str(body.get("username") or "").strip()
password = str(body.get("password") or "")
with db() as conn:
user = conn.execute(
"SELECT * FROM users WHERE username = ?", (username,)
).fetchone()
if not user or not _verify_password(password, user["password_hash"]):
self._record_login_fail(ip)
return self._json(HTTPStatus.UNAUTHORIZED, {"error": "invalid credentials"})
token = secrets.token_urlsafe(32)
csrf = secrets.token_urlsafe(24)
conn.execute(
"INSERT INTO sessions (token, user_id, csrf, expires_at, created_at) VALUES (?, ?, ?, ?, ?)",
(token, user["id"], csrf, _now() + SESSION_TTL, _now()),
)
self._clear_login_fails(ip)
out = json.dumps(
{"ok": True, "username": username, "csrf": csrf}, ensure_ascii=False
).encode("utf-8")
self.send_response(HTTPStatus.OK)
self._set_session_cookie(token, SESSION_TTL)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(out)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(out)
# --- API PUT / DELETE -------------------------------------------------------
def _api_put(self, path: str):
sess = self._require_session()
if not sess:
return
if not self._require_csrf(sess):
return
try:
body = self._read_json()
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path.startswith("/api/categories/"):
slug = path.rsplit("/", 1)[-1]
body["slug"] = body.get("slug") or slug
try:
with db() as conn:
exists = conn.execute(
"SELECT id FROM categories WHERE slug = ?", (slug,)
).fetchone()
if not exists:
return self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
# allow slug rename via body.slug
if body["slug"] != slug:
conn.execute("DELETE FROM categories WHERE slug = ?", (slug,))
upsert_category(conn, body)
row = conn.execute(
"SELECT * FROM categories WHERE slug = ?", (body["slug"],)
).fetchone()
return self._json(HTTPStatus.OK, {"category": category_dict(row)})
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
if path.startswith("/api/products/"):
pid = path.rsplit("/", 1)[-1]
try:
with db() as conn:
exists = conn.execute(
"SELECT * FROM products WHERE id = ?", (pid,)
).fetchone()
if not exists:
return self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
if "variants" in body and "colorVariants" not in body:
body["colorVariants"] = body.pop("variants")
data = normalize_product_input({**product_dict(exists), **body}, existing_id=pid)
data["id"] = pid
clash = conn.execute(
"SELECT id FROM products WHERE slug = ? AND id != ?",
(data["slug"], pid),
).fetchone()
if clash:
return self._json(
HTTPStatus.CONFLICT, {"error": "slug already exists"}
)
upsert_product(conn, data)
row = conn.execute(
"SELECT * FROM products WHERE id = ?", (pid,)
).fetchone()
return self._json(HTTPStatus.OK, {"product": product_dict(row)})
except Exception as exc:
return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
def _api_delete(self, path: str):
sess = self._require_session()
if not sess:
return
if not self._require_csrf(sess):
return
if path.startswith("/api/categories/"):
slug = path.rsplit("/", 1)[-1]
with db() as conn:
conn.execute("DELETE FROM categories WHERE slug = ?", (slug,))
return self._json(HTTPStatus.OK, {"ok": True})
if path.startswith("/api/products/"):
pid = path.rsplit("/", 1)[-1]
with db() as conn:
conn.execute("DELETE FROM products WHERE id = ?", (pid,))
return self._json(HTTPStatus.OK, {"ok": True})
self._json(HTTPStatus.NOT_FOUND, {"error": "not found"})
def main():
parser = argparse.ArgumentParser(description="Kikomiko admin CMS")
parser.add_argument(
"--host",
default=os.environ.get("ADMIN_HOST", "127.0.0.1"),
)
parser.add_argument(
"--port",
type=int,
default=int(os.environ.get("ADMIN_PORT", "8090")),
)
args = parser.parse_args()
init_db()
print(f"Catalog export path → {DIST_CATALOG}", flush=True)
httpd = ThreadingHTTPServer((args.host, args.port), AdminHandler)
print(f"Admin CMS → http://{args.host}:{args.port}/", flush=True)
try:
httpd.serve_forever()
except KeyboardInterrupt:
print("\nstopped", flush=True)
if __name__ == "__main__":
main()