#!/usr/bin/env python3 """Kikomiko admin CMS — Python stdlib + SQLite. Run: python3 server.py --port 8090""" from __future__ import annotations import argparse import hashlib import hmac import json import math import os import re import secrets import shutil import sqlite3 import sys import tempfile import time import uuid from http import HTTPStatus from http.cookies import SimpleCookie from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from urllib.parse import parse_qs, urlparse from ai_translate import ( category_needs_fa, collect_jobs, load_settings as load_ai_settings, public_settings as public_ai_settings, resolve_category_api_key, save_settings as save_ai_settings, translate_category_name, translate_product, ) from detail_schema import ( default_catalog_detail_queries, ensure_product_queries, fill_details_via_serp, missing_detail_keys, ) SQLITE_HEADER = b"SQLite format 3\x00" DB_UPLOAD_MAX = 80_000_000 ROOT = Path(__file__).resolve().parent DATA_DIR = ROOT / "data" STATIC_DIR = ROOT / "static" DB_PATH = DATA_DIR / "kikomiko.db" CREDENTIALS_PATH = DATA_DIR / ".admin-credentials" SEED_CATEGORIES = ROOT / "seed_categories.json" DIST_CATALOG = Path( os.environ.get( "KIKOMIKO_CATALOG_PATH", str(ROOT.parent / "dist" / "data" / "catalog.dedupe2.json"), ) ) SESSION_TTL = 60 * 60 * 12 LOGIN_WINDOW = 900 LOGIN_MAX_FAILS = 8 PBKDF2_ITERS = 310_000 # Persian labels for KIKO path segments / seed category names (admin + catalog). KIKO_SEGMENT_FA: dict[str, str] = { "MAKE-UP": "آرایش", "EYES MAKE-UP": "چشم", "FACE MAKE-UP": "صورت", "LIPS MAKE-UP": "لب", "NAILS MAKE-UP": "ناخن", "SKIN CARE": "مراقبت پوست", "FRAGRANCES": "عطر", "ACCESSORIES": "اکسسوری", "EYE PENCILS": "مداد چشم", "EYEBROWS": "ابرو", "EYELINER": "خط چشم", "EYESHADOWS": "سایه چشم", "MASCARAS": "ریمل", "BLUSH": "رژگونه", "BRONZERS": "برنزر", "CONCEALERS": "کانسیلر", "CONTOURING": "کانتور", "FOUNDATION": "کرم پودر", "HIGHLIGHTERS FACE": "هایلایتر", "POWDERS": "پودر", "PRIMERS AND FIXERS FACE": "پرایمر و فیکس", "LIP GLOSS": "لیپ‌گلاس", "LIP PENCILS": "مداد لب", "LIPS MAKE UP KIT": "ست لب", "LIPSTICK": "رژ لب", "MARKER": "مارکر لب", "NAIL LACQUER": "لاک ناخن", "FACE SKIN CARE": "مراقبت صورت", "LIP CARE": "مراقبت لب", "OTHER SKINCARE": "مراقبت پوست", "SUN CARE": "ضدآفتاب", "FULL SIZE FRAG": "عطر", "EYE ACCESSORIES": "اکسسوری چشم", "MAKE-UP BAG": "کیف آرایشی", "BRUSHES": "براش", "FACE ACCESSORIES": "اکسسوری صورت", "HAND AND FOOT ACCESSORIES": "اکسسوری دست و پا", "SKIN CARE KIT": "ست مراقبت پوست", "FACE MAKE-UP KIT": "ست آرایش صورت", "EYES MAKE-UP KIT": "ست آرایش چشم", "FACE CLEASING": "پاک‌کننده صورت", "FACE CLEANSING": "پاک‌کننده صورت", "MASK": "ماسک", "ACCESSORIES KIT": "ست اکسسوری", "HAIRCARE": "مراقبت مو", "HAIR ACCESSORIES": "اکسسوری مو", "SERVICE": "سرویس", "EMPTY CONTAINER": "ظرف خالی", "PRIMERS AND FIXERS EYES": "پرایمر و فیکس چشم", "PRIMERS AND FIXERS LIPS": "پرایمر و فیکس لب", "PRIMERS AND FIXERS FACE": "پرایمر و فیکس صورت", "FIXER & THINNER": "فیکسر و تینر", "FIXER AND THINNER": "فیکسر و تینر", "MIXING SOLUTION": "محلول ترکیب", "FRENCH MANICURE": "فرنچ مانیکور", "FULL SIZE FRAG": "عطر", "FULL SIZE HAIR": "مراقبت مو", "MAKE UP": "آرایش", "MAKE-UP": "آرایش", "SKIN CARE": "مراقبت پوست", "EYES MAKE-UP": "چشم", "FACE MAKE-UP": "صورت", "LIPS MAKE-UP": "لب", "NAILS MAKE-UP": "ناخن", "REMOVER": "پاک‌کننده", "SAMPLE HAIR": "نمونه مو", } def persian_category_label(category: str | None) -> str: """Most specific Persian label for a KIKO category path. Returns '' when no Persian mapping exists (never echo English as nameFa). """ parts = [p.strip() for p in str(category or "").split("/") if p.strip()] for part in reversed(parts): key = re.sub(r"\s+", " ", part).upper() if key in KIKO_SEGMENT_FA: return KIKO_SEGMENT_FA[key] # Also accept slug-like tokens: face-make-up → FACE MAKE-UP slugish = re.sub(r"[-_]+", " ", str(category or "")).strip().upper() if slugish in KIKO_SEGMENT_FA: return KIKO_SEGMENT_FA[slugish] return "" def seed_name_fa_by_slug() -> dict[str, str]: out: dict[str, str] = {} if SEED_CATEGORIES.is_file(): try: for item in json.loads(SEED_CATEGORIES.read_text(encoding="utf-8")): slug = str(item.get("slug") or "").strip() fa = str(item.get("name_fa") or item.get("nameFa") or "").strip() if slug and fa: out[slug] = fa except Exception: pass return out sys.path.insert(0, str(ROOT)) from fetchers import get_fetcher, list_fetchers # noqa: E402 from kiko_enrich import ( # noqa: E402 _family_key_from_product_url, discover_catalog_product_urls, fetch_kiko_product, merge_enrichment, ) # --- password / session helpers ------------------------------------------------- def _hash_password(password: str, salt: bytes | None = None) -> str: salt = salt or secrets.token_bytes(16) dk = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, PBKDF2_ITERS) return f"pbkdf2_sha256${PBKDF2_ITERS}${salt.hex()}${dk.hex()}" def _verify_password(password: str, stored: str) -> bool: try: algo, iters_s, salt_hex, hash_hex = stored.split("$", 3) if algo != "pbkdf2_sha256": return False iters = int(iters_s) salt = bytes.fromhex(salt_hex) expected = bytes.fromhex(hash_hex) got = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iters) return hmac.compare_digest(got, expected) except Exception: return False def _now() -> int: return int(time.time()) # --- database ------------------------------------------------------------------- class ClosingConnection(sqlite3.Connection): def __exit__(self, *args): try: return super().__exit__(*args) finally: self.close() def db() -> sqlite3.Connection: DATA_DIR.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(DB_PATH, factory=ClosingConnection) conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") conn.execute("PRAGMA journal_mode = WAL") return conn def init_db() -> None: with db() as conn: conn.executescript( """ CREATE TABLE IF NOT EXISTS users ( id INTEGER PRIMARY KEY, username TEXT NOT NULL UNIQUE COLLATE NOCASE, password_hash TEXT NOT NULL, created_at INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS sessions ( token TEXT PRIMARY KEY, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, csrf TEXT NOT NULL, expires_at INTEGER NOT NULL, created_at INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS categories ( id INTEGER PRIMARY KEY, slug TEXT NOT NULL UNIQUE, name TEXT NOT NULL, name_fa TEXT NOT NULL DEFAULT '', parent_slug TEXT, sort_order INTEGER NOT NULL DEFAULT 0, active INTEGER NOT NULL DEFAULT 1, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS products ( id TEXT PRIMARY KEY, slug TEXT NOT NULL UNIQUE, name TEXT NOT NULL, name_fa TEXT NOT NULL DEFAULT '', price REAL NOT NULL DEFAULT 0, compare_at_price REAL, stock INTEGER NOT NULL DEFAULT 0, hero_image TEXT, category_slug TEXT, category_fa TEXT NOT NULL DEFAULT '', description TEXT NOT NULL DEFAULT '', sku TEXT, external_id TEXT, source TEXT NOT NULL DEFAULT 'manual', color_variants TEXT NOT NULL DEFAULT '[]', active INTEGER NOT NULL DEFAULT 1, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_products_category ON products(category_slug); CREATE INDEX IF NOT EXISTS idx_products_external ON products(external_id); CREATE INDEX IF NOT EXISTS idx_products_sku ON products(sku); CREATE TABLE IF NOT EXISTS login_attempts ( ip TEXT PRIMARY KEY, fails INTEGER NOT NULL DEFAULT 0, window_start INTEGER NOT NULL ); """ ) columns = {r["name"] for r in conn.execute("PRAGMA table_info(products)")} if "content_json" not in columns: conn.execute("ALTER TABLE products ADD COLUMN content_json TEXT NOT NULL DEFAULT '{}'") conn.execute("CREATE TABLE IF NOT EXISTS catalog_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)") ensure_admin_user() def ensure_admin_user() -> None: username = os.environ.get("ADMIN_USER", "admin").strip() or "admin" password = os.environ.get("ADMIN_PASS") with db() as conn: row = conn.execute("SELECT id FROM users WHERE username = ?", (username,)).fetchone() if row: return if not password: password = secrets.token_urlsafe(16) CREDENTIALS_PATH.write_text( f"username={username}\npassword={password}\n" "Change this password after first login.\n", encoding="utf-8", ) os.chmod(CREDENTIALS_PATH, 0o600) print( f"\n=== ADMIN CREDENTIALS (first run) ===\n" f"username: {username}\npassword: {password}\n" f"also saved to {CREDENTIALS_PATH}\n" f"====================================\n", flush=True, ) conn.execute( "INSERT INTO users (username, password_hash, created_at) VALUES (?, ?, ?)", (username, _hash_password(password), _now()), ) # --- row helpers ---------------------------------------------------------------- def category_dict(row: sqlite3.Row) -> dict: return { "id": row["id"], "slug": row["slug"], "name": row["name"], "nameFa": row["name_fa"], "parentSlug": row["parent_slug"], "sortOrder": row["sort_order"], "active": bool(row["active"]), "createdAt": row["created_at"], "updatedAt": row["updated_at"], } def product_dict(row: sqlite3.Row) -> dict: try: variants = json.loads(row["color_variants"] or "[]") except json.JSONDecodeError: variants = [] return { **json.loads(row["content_json"] or "{}"), "id": row["id"], "slug": row["slug"], "name": row["name"], "nameFa": row["name_fa"], "price": row["price"], "compareAtPrice": row["compare_at_price"], "stock": row["stock"], "heroImage": row["hero_image"], "category": row["category_slug"], "categoryFa": row["category_fa"], "description": row["description"], "sku": row["sku"], "externalId": row["external_id"], "source": row["source"], "colorVariants": variants, "active": bool(row["active"]), "createdAt": row["created_at"], "updatedAt": row["updated_at"], } def catalog_product(row: sqlite3.Row) -> dict: # Export the complete product, including descriptions, tables, galleries, and detailQueries. return ensure_product_queries(product_dict(row)) def _known_kiko_families() -> set[str]: """Family keys already present locally (from sourceUrl or local slug).""" known: set[str] = set() with db() as conn: rows = conn.execute( "SELECT content_json, slug FROM products WHERE active = 1" ).fetchall() for row in rows: try: cj = json.loads(row["content_json"] or "{}") except json.JSONDecodeError: cj = {} src = str(cj.get("sourceUrl") or "").strip() if src and "/p/" in src: known.add(_family_key_from_product_url(src)) slug = str(row["slug"] or "").strip().lower() if slug: # strip trailing shade-sku pattern if present m = re.match(r"^(.+)-(\d+)-(\d+)$", slug) known.add(m.group(1) if m else slug) return known def _slugify(value: str) -> str: s = (value or "").strip().lower() s = re.sub(r"[^\w\s-]", "", s, flags=re.UNICODE) s = re.sub(r"[-\s]+", "-", s).strip("-") return s or f"item-{secrets.token_hex(3)}" def _pick(d: dict, *keys, default=None): for k in keys: if k in d and d[k] is not None: return d[k] return default def normalize_variants(raw_variants, *, product_id: str, product_name: str, product_slug: str) -> list: """Normalize color/shade variants to the storefront colorVariants shape.""" if raw_variants is None: return [] if isinstance(raw_variants, str): raw_variants = json.loads(raw_variants) if raw_variants.strip() else [] if not isinstance(raw_variants, list): raise ValueError("colorVariants must be a list") out = [] seen_slugs = set() seen_ids = set() for i, raw in enumerate(raw_variants): if not isinstance(raw, dict): raise ValueError("Each variant must be an object") validate_content(raw) shade = str(_pick(raw, "shade", "shadeCode", "label", "color", default="") or "").strip() if not shade: shade = " ".join(str(raw[k]) for k in ("colorCode", "colorName") if raw.get(k)) if not shade and not _pick(raw, "slug", "id", default=None): continue vid = str(_pick(raw, "id", default="") or "").strip() or f"{product_id}-v{i+1}" if vid in seen_ids: raise ValueError("Duplicate variant id: " + vid) seen_ids.add(vid) vslug = str(_pick(raw, "slug", default="") or "").strip() if not vslug: base = _slugify(f"{product_slug}-{shade}" if shade else f"{product_slug}-v{i+1}") vslug = base else: vslug = _slugify(vslug) # keep slugs unique inside one product original = vslug n = 2 while vslug in seen_slugs: vslug = f"{original}-{n}" n += 1 seen_slugs.add(vslug) compare = _pick(raw, "compareAtPrice", "compare_at_price", default=None) compare = float(compare) if compare not in (None, "") else None images = _pick(raw, "images", "gallery", default=[]) or [] if isinstance(images, str): images = [x.strip() for x in images.split(",") if x.strip()] if not isinstance(images, list): images = [] out.append( { **raw, "id": vid, "slug": vslug, "name": str(_pick(raw, "name", default="") or product_name).strip() or product_name, "shade": shade or str(raw.get("shadeCode") or raw.get("colorName") or raw.get("colorCode") or vslug), "price": float(_pick(raw, "price", default=0) or 0), "compareAtPrice": compare, "stock": int(_pick(raw, "stock", default=0) or 0), "heroImage": _pick(raw, "heroImage", "hero_image", default=None) or None, "sku": _pick(raw, "sku", default=None) or None, "externalId": _pick(raw, "externalId", "external_id", default=None) or None, "colorHex": _pick(raw, "colorHex", "color_hex", "hex", default=None) or None, "images": [str(x) for x in images if x], } ) return out def validate_content(raw): for key in ("price", "compareAtPrice", "stock"): if raw.get(key) not in (None, ""): value = float(raw[key]) if not math.isfinite(value) or value < 0 or (key == "stock" and not value.is_integer()): raise ValueError(key + " must be a finite nonnegative number (integer for stock)") for key in ("details", "specifications"): if key in raw and not isinstance(raw[key], (list, dict)): raise ValueError(key + " must be an array of label/value rows or an object") if isinstance(raw.get(key), list) and any(not isinstance(r, dict) or not (r.get("label") or r.get("key")) or "value" not in r for r in raw[key]): raise ValueError(key + " rows require label (or key) and value") if "detailQueries" in raw and raw["detailQueries"] is not None and not isinstance(raw["detailQueries"], dict): raise ValueError("detailQueries must be an object of key → search query strings") if "blocks" in raw and not isinstance(raw["blocks"], list): raise ValueError("blocks must be an array") def normalize_product_input(raw: dict, existing_id: str | None = None) -> dict: validate_content(raw) # Normalize details + attach scalable SerpAPI/detail queries before persist/export. ensure_product_queries(raw) name = str(_pick(raw, "name", default="") or "").strip() if not name: raise ValueError("name is required") slug = str(_pick(raw, "slug", default="") or "").strip() or _slugify(name) slug = _slugify(slug) pid = str(_pick(raw, "id", default="") or existing_id or uuid.uuid4().hex) variants = normalize_variants( _pick(raw, "colorVariants", "variants", "color_variants", default=[]), product_id=pid, product_name=name, product_slug=slug, ) price = float(_pick(raw, "price", default=0) or 0) compare = _pick(raw, "compareAtPrice", "compare_at_price", default=None) compare = float(compare) if compare not in (None, "") else None stock = int(_pick(raw, "stock", default=0) or 0) # Variants drive stock total; fill missing base price/image from primary shade. hero_image = _pick(raw, "heroImage", "hero_image", default=None) if variants: primary = next((v for v in variants if int(v.get("stock") or 0) > 0), variants[0]) stock = sum(int(v.get("stock") or 0) for v in variants) if price == 0 and primary.get("price"): price = float(primary["price"] or 0) if compare is None and primary.get("compareAtPrice") is not None: compare = primary["compareAtPrice"] if not hero_image and primary.get("heroImage"): hero_image = primary["heroImage"] active = _pick(raw, "active", default=True) if isinstance(active, str): active = active.lower() in ("1", "true", "yes", "on") return { "id": pid, "slug": slug, "name": name, "name_fa": str(_pick(raw, "nameFa", "name_fa", default="") or ""), "price": price, "compare_at_price": compare, "stock": stock, "hero_image": hero_image, "category_slug": _pick(raw, "category", "category_slug", default=None) or None, "category_fa": ( str(_pick(raw, "categoryFa", "category_fa", default="") or "").strip() or persian_category_label( str(_pick(raw, "category", "category_slug", default="") or "") ) ), "description": str(_pick(raw, "description", default="") or ""), "sku": _pick(raw, "sku", default=None) or None, "external_id": _pick(raw, "externalId", "external_id", default=None) or None, "source": str(_pick(raw, "source", default="manual") or "manual"), "color_variants": json.dumps(variants, ensure_ascii=False), "content_json": json.dumps({k: v for k, v in raw.items() if k not in ("variants", "colorVariants", "color_variants")}, ensure_ascii=False), "active": 1 if active else 0, } def upsert_product(conn: sqlite3.Connection, data: dict) -> None: ts = _now() conn.execute( """ INSERT INTO products ( id, slug, name, name_fa, price, compare_at_price, stock, hero_image, category_slug, category_fa, description, sku, external_id, source, color_variants, active, created_at, updated_at ) VALUES ( :id, :slug, :name, :name_fa, :price, :compare_at_price, :stock, :hero_image, :category_slug, :category_fa, :description, :sku, :external_id, :source, :color_variants, :active, :created_at, :updated_at ) ON CONFLICT(id) DO UPDATE SET slug=excluded.slug, name=excluded.name, name_fa=excluded.name_fa, price=excluded.price, compare_at_price=excluded.compare_at_price, stock=excluded.stock, hero_image=excluded.hero_image, category_slug=excluded.category_slug, category_fa=excluded.category_fa, description=excluded.description, sku=excluded.sku, external_id=excluded.external_id, source=excluded.source, color_variants=excluded.color_variants, active=excluded.active, updated_at=excluded.updated_at """, {**data, "created_at": ts, "updated_at": ts}, ) conn.execute("UPDATE products SET content_json = ? WHERE id = ?", (data["content_json"], data["id"])) def upsert_category(conn: sqlite3.Connection, raw: dict) -> None: slug = str(raw.get("slug") or "").strip() if not slug: raise ValueError("slug is required") name = str(raw.get("name") or slug).strip() name_fa = str(raw.get("name_fa") or raw.get("nameFa") or "").strip() if not name_fa: seed_fa = seed_name_fa_by_slug().get(slug, "") name_fa = seed_fa or persian_category_label(name) or persian_category_label( name.replace("-", " ") ) parent = raw.get("parent_slug") if "parent_slug" in raw else raw.get("parentSlug") parent = (str(parent).strip() or None) if parent else None sort_order = int(raw.get("sort_order", raw.get("sortOrder", 0)) or 0) active = raw.get("active", True) if isinstance(active, str): active = active.lower() in ("1", "true", "yes", "on") ts = _now() conn.execute( """ INSERT INTO categories (slug, name, name_fa, parent_slug, sort_order, active, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(slug) DO UPDATE SET name=excluded.name, name_fa=CASE WHEN excluded.name_fa != '' THEN excluded.name_fa ELSE categories.name_fa END, parent_slug=excluded.parent_slug, sort_order=excluded.sort_order, active=excluded.active, updated_at=excluded.updated_at """, (slug, name, name_fa, parent, sort_order, 1 if active else 0, ts, ts), ) def find_existing_product(conn: sqlite3.Connection, raw: dict): """Resolve an existing family for upsert so KIKO exports do not create duplicates.""" rid = raw.get("id") if rid: row = conn.execute("SELECT * FROM products WHERE id = ?", (str(rid),)).fetchone() if row: return row slug = str(raw.get("slug") or "").strip() if slug: row = conn.execute("SELECT * FROM products WHERE slug = ?", (slug,)).fetchone() if row: return row sku = str(raw.get("sku") or "").strip() if sku: row = conn.execute( "SELECT * FROM products WHERE sku = ? AND active = 1 ORDER BY length(description) DESC LIMIT 1", (sku,), ).fetchone() if row: return row external = str(raw.get("externalId") or raw.get("external_id") or "").strip() if external: row = conn.execute( "SELECT * FROM products WHERE external_id = ? AND active = 1 LIMIT 1", (external,), ).fetchone() if row: return row match = raw.get("match") if isinstance(raw.get("match"), dict) else {} for key in match.get("slugs") or []: key = str(key or "").strip() if not key: continue row = conn.execute("SELECT * FROM products WHERE slug = ?", (key,)).fetchone() if row: return row name = str(raw.get("name") or match.get("name") or "").strip() if name: row = conn.execute( "SELECT * FROM products WHERE active = 1 AND lower(name) = ? LIMIT 1", (name.lower(),), ).fetchone() if row: return row # "Love Fusion … Foundation" should match "Love Fusion … Foundation 0NR" row = conn.execute( """ SELECT * FROM products WHERE active = 1 AND lower(name) LIKE ? ORDER BY length(description) DESC, length(color_variants) DESC LIMIT 1 """, (name.lower() + " %",), ).fetchone() if row: return row return None def import_products_payload(payload) -> dict: if isinstance(payload, dict): products = payload.get("products") if products is None: raise ValueError("JSON must contain a products array") elif isinstance(payload, list): products = payload else: raise ValueError("invalid JSON") if not isinstance(products, list): raise ValueError("products must be an array") created = updated = skipped = 0 errors = [] with db() as conn: if isinstance(payload, dict): for key in ("metadata", "images"): if key in payload: value = payload[key] if key == "metadata": if not isinstance(value, dict): raise ValueError("metadata must be an object") prior = conn.execute("SELECT value FROM catalog_metadata WHERE key='metadata'").fetchone() value = {**(json.loads(prior[0]) if prior else {}), **value} if key == "images": if not isinstance(value, list): raise ValueError("images must be an array") prior = conn.execute("SELECT value FROM catalog_metadata WHERE key='images'").fetchone() merged = {x.get("original_url") or x.get("id"): x for x in (json.loads(prior[0]) if prior else [])} for x in value: if not isinstance(x, dict) or not (x.get("original_url") or x.get("id")): raise ValueError("image mapping requires original_url or id") merged[x.get("original_url") or x.get("id")] = x value = list(merged.values()) conn.execute("INSERT INTO catalog_metadata VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, json.dumps(value, ensure_ascii=False))) for i, raw in enumerate(products): if not isinstance(raw, dict): skipped += 1 continue try: existing = find_existing_product(conn, raw) if existing: incoming = dict(raw) if "variants" in incoming and "colorVariants" not in incoming: incoming["colorVariants"] = incoming.pop("variants") if "color_variants" in incoming and "colorVariants" not in incoming: incoming["colorVariants"] = incoming.pop("color_variants") # Keep local Persian/SEO text when incoming description is English source fill. prior = product_dict(existing) for key in ("description", "seoTitle", "seoDescription", "nameFa"): incoming_val = incoming.get(key) prior_val = prior.get(key) if (prior_val or "").strip() and ( not (incoming_val or "").strip() or incoming.get("source") == "kikocosmetics" ): # Do not overwrite reviewed local copy with KIKO English on upsert. if key == "description" and incoming.get("source") == "kikocosmetics": incoming["sourceDescription"] = incoming.get("sourceDescription") or incoming_val incoming[key] = prior_val elif (prior_val or "").strip(): incoming[key] = prior_val # Keep existing storefront slug/id so we never create a second family page. raw = { **prior, **incoming, "id": existing["id"], "slug": prior.get("slug") or incoming.get("slug"), } data = normalize_product_input(raw, existing["id"] if existing else None) # Prefer slug uniqueness when colliding with different id clash = conn.execute( "SELECT id FROM products WHERE slug = ? AND id != ?", (data["slug"], data["id"]), ).fetchone() if clash: data["slug"] = f"{data['slug']}-{data['id'][:6]}" upsert_product(conn, data) if existing: updated += 1 else: created += 1 except Exception as exc: errors.append({"index": i, "error": str(exc)}) skipped += 1 return {"created": created, "updated": updated, "skipped": skipped, "errors": errors[:50]} def apply_stock_updates(updates: list) -> dict: matched = updated = 0 with db() as conn: for u in updates: if not isinstance(u, dict) or u.get("stock") is None: continue stock = int(u["stock"]) row = None for key, col in ( ("id", "id"), ("external_id", "external_id"), ("externalId", "external_id"), ("sku", "sku"), ("slug", "slug"), ): val = u.get(key) if val: row = conn.execute( f"SELECT id FROM products WHERE {col} = ?", (str(val),) ).fetchone() if row: break if not row: continue matched += 1 conn.execute( "UPDATE products SET stock = ?, updated_at = ? WHERE id = ?", (stock, _now(), row["id"]), ) updated += 1 return {"matched": matched, "updated": updated} def export_catalog() -> dict: with db() as conn: rows = conn.execute( "SELECT * FROM products WHERE active = 1 ORDER BY updated_at DESC" ).fetchall() categories = [category_dict(r) for r in conn.execute("SELECT * FROM categories ORDER BY id")] extra = {r["key"]: json.loads(r["value"]) for r in conn.execute("SELECT * FROM catalog_metadata")} # Shared detail-query templates travel with the catalog for future imports/tools. templates = extra.get("detailQueryTemplates") or default_catalog_detail_queries() extra["detailQueryTemplates"] = templates with db() as conn: conn.execute( "INSERT INTO catalog_metadata VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", ("detailQueryTemplates", json.dumps(templates, ensure_ascii=False)), ) payload = { **extra, "version": "dedupe2", "schemaVersion": 2, "categories": categories, "products": [catalog_product(r) for r in rows], "exportedAt": _now(), } DIST_CATALOG.parent.mkdir(parents=True, exist_ok=True) atomic_json(DIST_CATALOG, payload) return {"path": str(DIST_CATALOG), "count": len(payload["products"])} def atomic_json(destination: Path, payload) -> None: destination.parent.mkdir(parents=True, exist_ok=True) fd, name = tempfile.mkstemp(dir=destination.parent, prefix=".catalog-", suffix=".json") try: with os.fdopen(fd, "w", encoding="utf-8") as stream: json.dump(payload, stream, ensure_ascii=False) stream.flush() os.fsync(stream.fileno()) os.chmod(name, 0o644) os.replace(name, destination) finally: if os.path.exists(name): os.unlink(name) def export_import_bundle(destination: Path) -> dict: """Private lossless interchange export; includes inactive products, no credentials.""" payload = import_bundle() atomic_json(destination, payload) return {"path": str(destination), "count": len(payload["products"])} def import_bundle() -> dict: with db() as conn: payload = {r["key"]: json.loads(r["value"]) for r in conn.execute("SELECT * FROM catalog_metadata")} payload.update(schemaVersion=2, products=[product_dict(r) for r in conn.execute("SELECT * FROM products ORDER BY id")], categories=[category_dict(r) for r in conn.execute("SELECT * FROM categories ORDER BY id")]) return payload def checkpoint_database() -> None: """Flush WAL into the main .db file so downloads are self-contained.""" if not DB_PATH.is_file(): return with db() as conn: conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") def validate_sqlite_backup(path: Path) -> dict: raw = path.read_bytes()[:16] if not raw.startswith(SQLITE_HEADER): raise ValueError("file is not a SQLite database") conn = sqlite3.connect(str(path)) try: tables = { r[0] for r in conn.execute( "SELECT name FROM sqlite_master WHERE type='table'" ).fetchall() } required = {"products", "categories", "users"} missing = required - tables if missing: raise ValueError( "missing required tables: " + ", ".join(sorted(missing)) ) products = conn.execute("SELECT COUNT(*) FROM products").fetchone()[0] categories = conn.execute("SELECT COUNT(*) FROM categories").fetchone()[0] users = conn.execute("SELECT COUNT(*) FROM users").fetchone()[0] return { "products": products, "categories": categories, "users": users, "bytes": path.stat().st_size, } finally: conn.close() def restore_database_bytes(data: bytes) -> dict: if len(data) < 100: raise ValueError("file too small") if not data.startswith(SQLITE_HEADER): raise ValueError("file is not a SQLite database") DATA_DIR.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile( prefix="kikomiko-restore-", suffix=".db", dir=DATA_DIR, delete=False ) as tmp: tmp.write(data) tmp_path = Path(tmp.name) try: stats = validate_sqlite_backup(tmp_path) checkpoint_database() bak = DATA_DIR / f"kikomiko.db.bak-{_now()}" if DB_PATH.is_file(): shutil.copy2(DB_PATH, bak) # Drop WAL sidecars so the restored main file is authoritative. for side in (DATA_DIR / "kikomiko.db-wal", DATA_DIR / "kikomiko.db-shm"): if side.is_file(): side.unlink() os.replace(tmp_path, DB_PATH) init_db() stats["backup"] = str(bak) if bak.is_file() else None return stats except Exception: if tmp_path.is_file(): tmp_path.unlink() raise # --- HTTP handler --------------------------------------------------------------- class AdminHandler(BaseHTTPRequestHandler): server_version = "KikomikoAdmin/1.0" def log_message(self, fmt, *args): sys.stderr.write("%s - %s\n" % (self.address_string(), fmt % args)) def _json(self, status: int, payload: dict): body = json.dumps(payload, ensure_ascii=False).encode("utf-8") self.send_response(status) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.send_header("Cache-Control", "no-store") self.send_header("X-Content-Type-Options", "nosniff") self.end_headers() self.wfile.write(body) def _text(self, status: int, text: str, content_type="text/plain; charset=utf-8"): body = text.encode("utf-8") self.send_response(status) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(body))) self.send_header("X-Content-Type-Options", "nosniff") self.end_headers() self.wfile.write(body) def _read_json(self) -> dict: length = int(self.headers.get("Content-Length") or 0) if length > 8_000_000: raise ValueError("payload too large") raw = self.rfile.read(length) if length else b"{}" if not raw: return {} data = json.loads(raw.decode("utf-8")) if not isinstance(data, dict): raise ValueError("JSON object required") return data def _cookies(self) -> SimpleCookie: c = SimpleCookie() if "Cookie" in self.headers: c.load(self.headers["Cookie"]) return c def _client_ip(self) -> str: return self.client_address[0] or "unknown" def _get_session(self): token = None cookies = self._cookies() if "session" in cookies: token = cookies["session"].value if not token: return None with db() as conn: row = conn.execute( """ SELECT s.token, s.csrf, s.expires_at, u.id AS user_id, u.username FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token = ? """, (token,), ).fetchone() if not row or row["expires_at"] < _now(): if row: conn.execute("DELETE FROM sessions WHERE token = ?", (token,)) return None # sliding expiry conn.execute( "UPDATE sessions SET expires_at = ? WHERE token = ?", (_now() + SESSION_TTL, token), ) return dict(row) def _require_session(self): sess = self._get_session() if not sess: self._json(HTTPStatus.UNAUTHORIZED, {"error": "unauthorized"}) return None return sess def _require_csrf(self, sess) -> bool: header = self.headers.get("X-CSRF-Token") or "" if not header or not hmac.compare_digest(header, sess["csrf"]): self._json(HTTPStatus.FORBIDDEN, {"error": "invalid csrf token"}) return False return True def _set_session_cookie(self, token: str, max_age: int): flags = f"session={token}; Path=/; HttpOnly; SameSite=Strict; Max-Age={max_age}" self.send_header("Set-Cookie", flags) def _clear_session_cookie(self): self.send_header( "Set-Cookie", "session=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0", ) def _login_allowed(self, ip: str) -> bool: with db() as conn: row = conn.execute( "SELECT fails, window_start FROM login_attempts WHERE ip = ?", (ip,) ).fetchone() if not row: return True if _now() - row["window_start"] > LOGIN_WINDOW: conn.execute("DELETE FROM login_attempts WHERE ip = ?", (ip,)) return True return row["fails"] < LOGIN_MAX_FAILS def _record_login_fail(self, ip: str): with db() as conn: row = conn.execute( "SELECT fails, window_start FROM login_attempts WHERE ip = ?", (ip,) ).fetchone() if not row or _now() - row["window_start"] > LOGIN_WINDOW: conn.execute( "INSERT OR REPLACE INTO login_attempts (ip, fails, window_start) VALUES (?, 1, ?)", (ip, _now()), ) else: conn.execute( "UPDATE login_attempts SET fails = fails + 1 WHERE ip = ?", (ip,) ) def _clear_login_fails(self, ip: str): with db() as conn: conn.execute("DELETE FROM login_attempts WHERE ip = ?", (ip,)) # --- routing ---------------------------------------------------------------- def do_GET(self): parsed = urlparse(self.path) path = parsed.path if path.startswith("/api/"): return self._api_get(path, parsed) return self._serve_static(path) def do_POST(self): parsed = urlparse(self.path) path = parsed.path if path.startswith("/api/"): return self._api_post(path) self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) def do_PUT(self): parsed = urlparse(self.path) if parsed.path.startswith("/api/"): return self._api_put(parsed.path) self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) def do_DELETE(self): parsed = urlparse(self.path) if parsed.path.startswith("/api/"): return self._api_delete(parsed.path) self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) def _serve_static(self, path: str): if path in ("", "/"): path = "/index.html" rel = path.lstrip("/") parts = Path(rel).parts if not parts or ".." in parts or parts[0].startswith("/"): return self._text(HTTPStatus.FORBIDDEN, "forbidden") # Join without resolving first so symlink hops (local fonts) stay allowed. file_path = STATIC_DIR.joinpath(*parts) if not file_path.is_file(): return self._text(HTTPStatus.NOT_FOUND, "not found") # Final path must stay inside the kikomiko project tree. resolved = file_path.resolve() project_root = ROOT.parent.resolve() try: resolved.relative_to(project_root) except ValueError: return self._text(HTTPStatus.FORBIDDEN, "forbidden") ext = resolved.suffix.lower() types = { ".html": "text/html; charset=utf-8", ".css": "text/css; charset=utf-8", ".js": "application/javascript; charset=utf-8", ".woff": "font/woff", ".woff2": "font/woff2", ".png": "image/png", ".svg": "image/svg+xml", ".json": "application/json; charset=utf-8", } data = resolved.read_bytes() self.send_response(HTTPStatus.OK) self.send_header("Content-Type", types.get(ext, "application/octet-stream")) self.send_header("Content-Length", str(len(data))) self.send_header("X-Content-Type-Options", "nosniff") if ext in (".html", ".js", ".css"): self.send_header("Cache-Control", "no-store") self.end_headers() self.wfile.write(data) # --- API GET ---------------------------------------------------------------- def _api_get(self, path: str, parsed): if path == "/api/health": return self._json(HTTPStatus.OK, {"ok": True}) if path == "/api/session": sess = self._get_session() if not sess: return self._json(HTTPStatus.OK, {"authenticated": False}) return self._json( HTTPStatus.OK, { "authenticated": True, "username": sess["username"], "csrf": sess["csrf"], }, ) sess = self._require_session() if not sess: return if path == "/api/export/json": return self._json(HTTPStatus.OK, import_bundle()) if path == "/api/categories": with db() as conn: rows = conn.execute( "SELECT * FROM categories ORDER BY sort_order, name_fa, name" ).fetchall() return self._json(HTTPStatus.OK, {"categories": [category_dict(r) for r in rows]}) if path == "/api/products": q = parse_qs(parsed.query) search = (q.get("q") or [""])[0].strip().lower() with db() as conn: rows = conn.execute( "SELECT * FROM products ORDER BY updated_at DESC LIMIT 2000" ).fetchall() products = [product_dict(r) for r in rows] if search: products = [ p for p in products if search in f"{p['name']} {p['slug']} {p['sku'] or ''} {p['category'] or ''}".lower() ] return self._json(HTTPStatus.OK, {"products": products}) if path.startswith("/api/products/"): pid = path.split("/", 3)[-1] with db() as conn: row = conn.execute("SELECT * FROM products WHERE id = ?", (pid,)).fetchone() if not row: return self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) return self._json(HTTPStatus.OK, {"product": product_dict(row)}) if path == "/api/fetchers": return self._json(HTTPStatus.OK, {"fetchers": list_fetchers()}) if path == "/api/stats": with db() as conn: pc = conn.execute("SELECT COUNT(*) AS c FROM products").fetchone()["c"] cc = conn.execute("SELECT COUNT(*) AS c FROM categories").fetchone()["c"] stock = conn.execute( "SELECT COALESCE(SUM(stock),0) AS s FROM products WHERE active=1" ).fetchone()["s"] return self._json( HTTPStatus.OK, {"products": pc, "categories": cc, "stockUnits": stock}, ) if path == "/api/settings/ai": return self._json(HTTPStatus.OK, public_ai_settings()) if path == "/api/export/database": if not DB_PATH.is_file(): return self._json(HTTPStatus.NOT_FOUND, {"error": "database not found"}) try: checkpoint_database() data = DB_PATH.read_bytes() except Exception as exc: return self._json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": str(exc)}) stamp = time.strftime("%Y%m%d-%H%M%S") filename = f"kikomiko-{stamp}.db" self.send_response(HTTPStatus.OK) self.send_header("Content-Type", "application/octet-stream") self.send_header( "Content-Disposition", f'attachment; filename="{filename}"' ) self.send_header("Content-Length", str(len(data))) self.send_header("Cache-Control", "no-store") self.send_header("X-Content-Type-Options", "nosniff") self.end_headers() self.wfile.write(data) return self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) # --- API POST --------------------------------------------------------------- def _api_post(self, path: str): if path == "/api/login": return self._handle_login() sess = self._require_session() if not sess: return if not self._require_csrf(sess): return if path == "/api/import/database": length = int(self.headers.get("Content-Length") or 0) if length <= 0: return self._json(HTTPStatus.BAD_REQUEST, {"error": "empty body"}) if length > DB_UPLOAD_MAX: return self._json( HTTPStatus.REQUEST_ENTITY_TOO_LARGE, {"error": f"file too large (max {DB_UPLOAD_MAX // 1_000_000}MB)"}, ) try: raw = self.rfile.read(length) result = restore_database_bytes(raw) return self._json(HTTPStatus.OK, {"ok": True, **result}) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) try: body = self._read_json() except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/logout": with db() as conn: conn.execute("DELETE FROM sessions WHERE token = ?", (sess["token"],)) self.send_response(HTTPStatus.OK) self._clear_session_cookie() body_out = json.dumps({"ok": True}).encode("utf-8") self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body_out))) self.end_headers() self.wfile.write(body_out) return if path == "/api/categories": try: with db() as conn: upsert_category(conn, body) row = conn.execute( "SELECT * FROM categories WHERE slug = ?", (body.get("slug"),), ).fetchone() return self._json(HTTPStatus.CREATED, {"category": category_dict(row)}) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/products": try: data = normalize_product_input(body) with db() as conn: clash = conn.execute( "SELECT id FROM products WHERE slug = ?", (data["slug"],) ).fetchone() if clash: return self._json( HTTPStatus.CONFLICT, {"error": "slug already exists"} ) upsert_product(conn, data) row = conn.execute( "SELECT * FROM products WHERE id = ?", (data["id"],) ).fetchone() return self._json(HTTPStatus.CREATED, {"product": product_dict(row)}) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/enrich/kiko": # Preview enrichment for the product editor (does not save). try: fetched = fetch_kiko_product( url=body.get("url"), query=body.get("query") or body.get("name"), market=str(body.get("market") or "en-gb"), ) return self._json(HTTPStatus.OK, {"product": fetched}) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path.startswith("/api/products/") and path.endswith("/enrich-kiko"): pid = path[len("/api/products/") : -len("/enrich-kiko")].strip("/") if not pid or "/" in pid: return self._json(HTTPStatus.BAD_REQUEST, {"error": "invalid product id"}) try: with db() as conn: exists = conn.execute( "SELECT * FROM products WHERE id = ?", (pid,) ).fetchone() if not exists: return self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) existing = product_dict(exists) fetched = fetch_kiko_product( url=body.get("url") or existing.get("sourceUrl"), query=body.get("query") or existing.get("displayName") or existing.get("name"), market=str(body.get("market") or existing.get("sourceMarket") or "en-gb"), ) merged = merge_enrichment( existing, fetched, overwrite_description=bool(body.get("overwriteDescription")), ) data = normalize_product_input(merged, existing_id=pid) data["id"] = pid upsert_product(conn, data) row = conn.execute( "SELECT * FROM products WHERE id = ?", (pid,) ).fetchone() return self._json( HTTPStatus.OK, { "product": product_dict(row), "sourceUrl": fetched.get("sourceUrl"), "detailsCount": len(fetched.get("details") or []), "variantCount": len(fetched.get("colorVariants") or []), }, ) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/import/categories": if not SEED_CATEGORIES.is_file(): return self._json(HTTPStatus.NOT_FOUND, {"error": "seed file missing"}) seed = json.loads(SEED_CATEGORIES.read_text(encoding="utf-8")) count = 0 with db() as conn: for item in seed: upsert_category(conn, item) count += 1 return self._json(HTTPStatus.OK, {"imported": count}) if path == "/api/import/catalog": source = body.get("source", "dist") if source == "dist": if not DIST_CATALOG.is_file(): return self._json( HTTPStatus.NOT_FOUND, {"error": f"catalog not found: {DIST_CATALOG}"}, ) payload = json.loads(DIST_CATALOG.read_text(encoding="utf-8")) elif source == "inline": payload = body.get("data") if payload is None: return self._json(HTTPStatus.BAD_REQUEST, {"error": "data required"}) elif source == "path": p = Path(body.get("path") or "").expanduser() if not p.is_file(): return self._json(HTTPStatus.BAD_REQUEST, {"error": "path not found"}) # Large catalog dumps (100MB+) — path import avoids the HTTP body limit. payload = json.loads(p.read_text(encoding="utf-8")) else: return self._json(HTTPStatus.BAD_REQUEST, {"error": "unknown source"}) cat_count = 0 if isinstance(payload, dict) and isinstance(payload.get("categories"), list): # Prefer parents before children cats = sorted( payload["categories"], key=lambda c: str((c or {}).get("full_path") or (c or {}).get("id") or "").count("/"), ) with db() as conn: for raw in cats: if not isinstance(raw, dict): continue name = str(raw.get("name") or raw.get("full_path") or "").strip() if not name: continue parent_id = raw.get("parent_id") or raw.get("parentSlug") parent_slug = None if parent_id: parent_slug = _slugify(str(parent_id).split(" / ")[-1].strip()) upsert_category( conn, { "slug": raw.get("slug") or _slugify(name), "name": name.title() if name == name.upper() else name, "name_fa": raw.get("name_fa") or raw.get("nameFa") or "", "parent_slug": parent_slug, "sort_order": raw.get("sort_order") or raw.get("sortOrder") or 0, "active": raw.get("active", True), }, ) cat_count += 1 result = import_products_payload(payload) result["categoriesImported"] = cat_count return self._json(HTTPStatus.OK, result) if path == "/api/export/catalog": try: result = export_catalog() return self._json(HTTPStatus.OK, result) except Exception as exc: return self._json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": str(exc)}) if path == "/api/fetch/run": fetcher_id = body.get("fetcher") action = body.get("action", "fetch") config = body.get("config") or {} try: module = get_fetcher(fetcher_id) if action == "fetch": rows = module.fetch_products(config) result = import_products_payload(rows) result["fetched"] = len(rows) return self._json(HTTPStatus.OK, result) if action == "sync": with db() as conn: products = [product_dict(r) for r in conn.execute("SELECT * FROM products")] updates = module.sync_stocks(config, products) result = apply_stock_updates(updates) result["received"] = len(updates) return self._json(HTTPStatus.OK, result) return self._json(HTTPStatus.BAD_REQUEST, {"error": "action must be fetch or sync"}) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/enrich/kiko-discover": # List product family URLs from the official KIKO sitemap. try: market = str(body.get("market") or "en-gb").strip() or "en-gb" one_per_family = body.get("onePerFamily", True) discovered = discover_catalog_product_urls( market, one_per_family=bool(one_per_family) ) skip_existing = body.get("skipExisting", True) urls = list(discovered.get("urls") or []) already = 0 if skip_existing: known_families = _known_kiko_families() filtered = [] for u in urls: if _family_key_from_product_url(u) in known_families: already += 1 continue filtered.append(u) urls = filtered return self._json( HTTPStatus.OK, { "ok": True, "market": discovered.get("market"), "sitemapUrl": discovered.get("sitemapUrl"), "shadeUrls": discovered.get("shadeUrls"), "familyUrls": discovered.get("familyUrls"), "alreadyPresent": already, "pending": len(urls), "urls": urls, "hint": "این لیست را می‌توانید وارد کنید یا دکمه ورود همه را بزنید.", }, ) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/enrich/kiko-bulk": # Bulk pull from kikocosmetics.com (many URLs, refill existing, or full catalog). try: market = str(body.get("market") or "en-gb").strip() or "en-gb" overwrite_description = bool(body.get("overwriteDescription")) limit = int(body.get("limit") or 30) limit = max(1, min(limit, 80)) mode = str(body.get("mode") or "urls").strip() or "urls" offset = max(0, int(body.get("offset") or 0)) jobs: list[dict] = [] catalog_meta: dict = {} if mode == "urls": urls = body.get("urls") if isinstance(urls, str): urls = [u.strip() for u in urls.replace(",", "\n").splitlines() if u.strip()] elif not isinstance(urls, list): urls = [] urls = [str(u).strip() for u in urls if str(u).strip()] # de-dupe, keep order seen = set() uniq = [] for u in urls: if u in seen: continue seen.add(u) uniq.append(u) for u in uniq[:limit]: jobs.append({"url": u}) catalog_meta = {"pendingAfter": max(0, len(uniq) - limit)} elif mode == "existing": only_missing = body.get("onlyMissing", True) with db() as conn: rows = conn.execute( "SELECT * FROM products WHERE active = 1 ORDER BY updated_at DESC" ).fetchall() for row in rows: p = product_dict(row) has_details = bool(p.get("details")) has_desc = bool((p.get("description") or "").strip()) if only_missing and has_details and has_desc: continue if not (p.get("sourceUrl") or p.get("name") or p.get("sku")): continue jobs.append( { "id": p.get("id"), "url": p.get("sourceUrl") or "", "query": p.get("name") or "", "existing": p, } ) if len(jobs) >= limit: break elif mode == "catalog": # Discover from sitemap, skip known products, take next batch. skip_existing = body.get("skipExisting", True) discovered = discover_catalog_product_urls(market, one_per_family=True) urls = list(discovered.get("urls") or []) already = 0 if skip_existing: known_families = _known_kiko_families() filtered = [] for u in urls: if _family_key_from_product_url(u) in known_families: already += 1 continue filtered.append(u) urls = filtered already = max(0, int(discovered.get("familyUrls") or 0) - len(urls)) batch = urls[offset : offset + limit] for u in batch: jobs.append({"url": u}) catalog_meta = { "shadeUrls": discovered.get("shadeUrls"), "familyUrls": discovered.get("familyUrls"), "alreadyPresent": already, "pendingTotal": len(urls), "offset": offset, "pendingAfter": max(0, len(urls) - (offset + len(batch))), "done": len(urls) <= (offset + len(batch)), } else: return self._json( HTTPStatus.BAD_REQUEST, {"error": "mode must be urls, existing, or catalog"}, ) if not jobs: return self._json( HTTPStatus.OK, { "ok": True, "mode": mode, "fetched": 0, "created": 0, "updated": 0, "errors": [], "done": True, "pendingAfter": 0, "hint": "چیزی برای دریافت نمانده.", **catalog_meta, }, ) fetched_rows = [] errors = [] for i, job in enumerate(jobs): try: fetched = fetch_kiko_product( url=job.get("url") or None, query=(job.get("query") or None) if not job.get("url") else None, market=market, ) existing = job.get("existing") if existing: fetched_rows.append( merge_enrichment( existing, fetched, overwrite_description=overwrite_description, ) ) else: fetched_rows.append(fetched) except Exception as exc: # noqa: BLE001 errors.append( { "url": job.get("url") or "", "query": job.get("query") or "", "error": str(exc), } ) if mode == "catalog" and i + 1 < len(jobs): time.sleep(0.35) result = import_products_payload(fetched_rows) if fetched_rows else { "created": 0, "updated": 0, "skipped": 0, "errors": [], } result.update( { "ok": True, "mode": mode, "fetched": len(fetched_rows), "jobs": len(jobs), "fetchErrors": errors[:40], "hint": "بعد از اتمام، از داشبورد «خروجی به catalog.json» را بزنید تا روی سایت بیاید.", **catalog_meta, } ) if "done" not in result: result["done"] = catalog_meta.get("pendingAfter", 0) == 0 return self._json(HTTPStatus.OK, result) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/password": current = body.get("current") or "" new_pass = body.get("newPassword") or "" if len(new_pass) < 10: return self._json( HTTPStatus.BAD_REQUEST, {"error": "new password must be at least 10 characters"}, ) with db() as conn: user = conn.execute( "SELECT * FROM users WHERE id = ?", (sess["user_id"],) ).fetchone() if not user or not _verify_password(current, user["password_hash"]): return self._json(HTTPStatus.FORBIDDEN, {"error": "current password wrong"}) conn.execute( "UPDATE users SET password_hash = ? WHERE id = ?", (_hash_password(new_pass), user["id"]), ) conn.execute("DELETE FROM sessions WHERE user_id = ?", (user["id"],)) return self._json(HTTPStatus.OK, {"ok": True, "relogin": True}) if path == "/api/settings/ai": try: kwargs = {} if "apiKey" in body: kwargs["api_key"] = str(body.get("apiKey") or "") if "baseUrl" in body: kwargs["base_url"] = str(body.get("baseUrl") or "") if "model" in body: kwargs["model"] = str(body.get("model") or "") if "serpApiKey" in body: kwargs["serp_api_key"] = str(body.get("serpApiKey") or "") if "categoryApiKey" in body: kwargs["category_api_key"] = str(body.get("categoryApiKey") or "") result = save_ai_settings(**kwargs) if kwargs else public_ai_settings() return self._json(HTTPStatus.OK, {"ok": True, **result}) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/translate/categories": try: # Local dictionary works without an API key; AI is only a fallback. settings = load_ai_settings() overwrite = bool(body.get("overwrite")) dry_run = bool(body.get("dryRun")) limit = int(body.get("limit") or 50) limit = max(1, min(limit, 200)) with db() as conn: rows = conn.execute( "SELECT * FROM categories ORDER BY sort_order, name" ).fetchall() candidates = [] for r in rows: cat = category_dict(r) if category_needs_fa(cat, overwrite=overwrite): candidates.append(cat) remaining = max(0, len(candidates) - limit) batch = candidates[:limit] results = [] translated = 0 errors = [] for cat in batch: if dry_run: results.append( { "slug": cat.get("slug"), "name": cat.get("name"), "nameFa": cat.get("nameFa") or "", } ) continue try: stats = translate_category_name( cat, overwrite=overwrite, settings=settings ) results.append(stats) if stats.get("changed"): # Force-write Persian label (bypass empty-preserve CASE). conn.execute( """ UPDATE categories SET name_fa = ?, updated_at = ? WHERE slug = ? """, (cat.get("nameFa") or "", _now(), cat["slug"]), ) translated += 1 except Exception as exc: # noqa: BLE001 errors.append(f"{cat.get('slug')}: {exc}") results.append( { "slug": cat.get("slug"), "changed": False, "errors": [str(exc)], } ) if any( code in str(exc) for code in ("HTTP 401", "HTTP 403", "HTTP 429") ): break return self._json( HTTPStatus.OK, { "ok": True, "dryRun": dry_run, "processed": len(results), "translatedCategories": translated, "remaining": remaining, "hasAiKey": bool(resolve_category_api_key(settings)), "errors": errors[:20], "results": results[:80], "hint": "Export catalog after translating so the storefront mega-menu updates.", }, ) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/enrich/details-serp": try: settings = load_ai_settings() serp_key = (settings.get("serpApiKey") or "").strip() if not serp_key: return self._json( HTTPStatus.BAD_REQUEST, {"error": "SerpAPI key is missing. Save it in Settings first."}, ) overwrite = bool(body.get("overwrite")) dry_run = bool(body.get("dryRun")) limit = int(body.get("limit") or 15) limit = max(1, min(limit, 50)) only_active = body.get("onlyActive", True) with db() as conn: if only_active: rows = conn.execute( "SELECT * FROM products WHERE active = 1 ORDER BY updated_at DESC" ).fetchall() else: rows = conn.execute( "SELECT * FROM products ORDER BY updated_at DESC" ).fetchall() candidates = [] for row in rows: product = ensure_product_queries(product_dict(row)) missing = missing_detail_keys(product, overwrite=overwrite) if missing: candidates.append((product, missing)) remaining = max(0, len(candidates) - limit) batch = candidates[:limit] results = [] filled_products = 0 filled_fields = 0 errors = [] for product, missing in batch: if dry_run: results.append( { "id": product.get("id"), "slug": product.get("slug"), "missing": missing, "queries": { k: (product.get("detailQueries") or {}).get(k) for k in missing }, } ) continue stats = fill_details_via_serp( product, api_key=serp_key, keys=missing, overwrite=overwrite, ) results.append(stats) if stats.get("errors"): errors.extend( f"{product.get('slug')}: {e}" for e in stats["errors"] ) if any( "HTTP 401" in e or "HTTP 403" in e or "HTTP 429" in e for e in stats["errors"] ): break if stats.get("filled"): data = normalize_product_input( product, existing_id=product["id"] ) data["id"] = product["id"] upsert_product(conn, data) filled_products += 1 filled_fields += len(stats["filled"]) return self._json( HTTPStatus.OK, { "ok": True, "dryRun": dry_run, "processed": len(results), "filledProducts": filled_products, "filledFields": filled_fields, "remaining": remaining, "errors": errors[:20], "results": results[:40], "hint": "Each product stores detailQueries in the catalog export for future imports.", }, ) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path == "/api/translate/descriptions": try: settings = load_ai_settings() if not (settings.get("apiKey") or "").strip(): return self._json( HTTPStatus.BAD_REQUEST, {"error": "AI API key is missing. Save it in Settings first."}, ) overwrite = bool(body.get("overwrite")) dry_run = bool(body.get("dryRun")) limit = int(body.get("limit") or 25) limit = max(1, min(limit, 100)) only_active = body.get("onlyActive", True) with db() as conn: if only_active: rows = conn.execute( "SELECT * FROM products WHERE active = 1 ORDER BY updated_at DESC" ).fetchall() else: rows = conn.execute( "SELECT * FROM products ORDER BY updated_at DESC" ).fetchall() candidates = [] for row in rows: product = product_dict(row) jobs = collect_jobs(product, overwrite=overwrite) if jobs: candidates.append(product) remaining = max(0, len(candidates) - limit) batch = candidates[:limit] results = [] translated_products = 0 translated_fields = 0 errors = [] for product in batch: if dry_run: jobs = collect_jobs(product, overwrite=overwrite) results.append( { "id": product.get("id"), "slug": product.get("slug"), "pending": [j[0] for j in jobs], } ) continue stats = translate_product( product, overwrite=overwrite, settings=settings ) results.append(stats) if stats.get("errors"): errors.extend( f"{product.get('slug')}: {e}" for e in stats["errors"] ) # Stop the whole batch on auth/quota style failures if any( "HTTP 401" in e or "HTTP 403" in e or "HTTP 429" in e for e in stats["errors"] ): break if stats.get("changed"): data = normalize_product_input(product, existing_id=product["id"]) data["id"] = product["id"] upsert_product(conn, data) translated_products += 1 translated_fields += len(stats["changed"]) return self._json( HTTPStatus.OK, { "ok": True, "dryRun": dry_run, "processed": len(results), "translatedProducts": translated_products, "translatedFields": translated_fields, "remaining": remaining, "errors": errors[:20], "results": results[:50], "hint": "Export catalog to storefront after translating.", }, ) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) def _handle_login(self): ip = self._client_ip() if not self._login_allowed(ip): return self._json( HTTPStatus.TOO_MANY_REQUESTS, {"error": "too many failed attempts; try later"}, ) try: body = self._read_json() except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) username = str(body.get("username") or "").strip() password = str(body.get("password") or "") with db() as conn: user = conn.execute( "SELECT * FROM users WHERE username = ?", (username,) ).fetchone() if not user or not _verify_password(password, user["password_hash"]): self._record_login_fail(ip) return self._json(HTTPStatus.UNAUTHORIZED, {"error": "invalid credentials"}) token = secrets.token_urlsafe(32) csrf = secrets.token_urlsafe(24) conn.execute( "INSERT INTO sessions (token, user_id, csrf, expires_at, created_at) VALUES (?, ?, ?, ?, ?)", (token, user["id"], csrf, _now() + SESSION_TTL, _now()), ) self._clear_login_fails(ip) out = json.dumps( {"ok": True, "username": username, "csrf": csrf}, ensure_ascii=False ).encode("utf-8") self.send_response(HTTPStatus.OK) self._set_session_cookie(token, SESSION_TTL) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(out))) self.send_header("Cache-Control", "no-store") self.end_headers() self.wfile.write(out) # --- API PUT / DELETE ------------------------------------------------------- def _api_put(self, path: str): sess = self._require_session() if not sess: return if not self._require_csrf(sess): return try: body = self._read_json() except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path.startswith("/api/categories/"): slug = path.rsplit("/", 1)[-1] body["slug"] = body.get("slug") or slug try: with db() as conn: exists = conn.execute( "SELECT id FROM categories WHERE slug = ?", (slug,) ).fetchone() if not exists: return self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) # allow slug rename via body.slug if body["slug"] != slug: conn.execute("DELETE FROM categories WHERE slug = ?", (slug,)) upsert_category(conn, body) row = conn.execute( "SELECT * FROM categories WHERE slug = ?", (body["slug"],) ).fetchone() return self._json(HTTPStatus.OK, {"category": category_dict(row)}) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) if path.startswith("/api/products/"): pid = path.rsplit("/", 1)[-1] try: with db() as conn: exists = conn.execute( "SELECT * FROM products WHERE id = ?", (pid,) ).fetchone() if not exists: return self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) if "variants" in body and "colorVariants" not in body: body["colorVariants"] = body.pop("variants") data = normalize_product_input({**product_dict(exists), **body}, existing_id=pid) data["id"] = pid clash = conn.execute( "SELECT id FROM products WHERE slug = ? AND id != ?", (data["slug"], pid), ).fetchone() if clash: return self._json( HTTPStatus.CONFLICT, {"error": "slug already exists"} ) upsert_product(conn, data) row = conn.execute( "SELECT * FROM products WHERE id = ?", (pid,) ).fetchone() return self._json(HTTPStatus.OK, {"product": product_dict(row)}) except Exception as exc: return self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) def _api_delete(self, path: str): sess = self._require_session() if not sess: return if not self._require_csrf(sess): return if path.startswith("/api/categories/"): slug = path.rsplit("/", 1)[-1] with db() as conn: conn.execute("DELETE FROM categories WHERE slug = ?", (slug,)) return self._json(HTTPStatus.OK, {"ok": True}) if path.startswith("/api/products/"): pid = path.rsplit("/", 1)[-1] with db() as conn: conn.execute("DELETE FROM products WHERE id = ?", (pid,)) return self._json(HTTPStatus.OK, {"ok": True}) self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) def main(): parser = argparse.ArgumentParser(description="Kikomiko admin CMS") parser.add_argument( "--host", default=os.environ.get("ADMIN_HOST", "127.0.0.1"), ) parser.add_argument( "--port", type=int, default=int(os.environ.get("ADMIN_PORT", "8090")), ) args = parser.parse_args() init_db() print(f"Catalog export path → {DIST_CATALOG}", flush=True) httpd = ThreadingHTTPServer((args.host, args.port), AdminHandler) print(f"Admin CMS → http://{args.host}:{args.port}/", flush=True) try: httpd.serve_forever() except KeyboardInterrupt: print("\nstopped", flush=True) if __name__ == "__main__": main()