first deployment of kikomiko with url photos
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
shayan khoshrou
co-authored by
Cursor
Cursor
parent
8d98304137
commit
ae3ed7c7b7
@@ -0,0 +1,335 @@
|
||||
"""Canonical product-detail keys + SerpAPI query templates for import/export.
|
||||
|
||||
Every product gets normalized `details` rows and a `detailQueries` map so future
|
||||
imports, exports, and SerpAPI enrichment stay consistent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
# Stable keys used on the PDP facts table (storefront + admin).
|
||||
DETAIL_LABELS: dict[str, str] = {
|
||||
"brand": "برند",
|
||||
"madeIn": "ساخت",
|
||||
"size": "حجم / سایز",
|
||||
"pao": "PAO",
|
||||
"finish": "فینیش",
|
||||
"coverage": "پوشش",
|
||||
"skinType": "نوع پوست",
|
||||
"skinTone": "تهرنگ پوست",
|
||||
"skinUndertone": "آندرتون",
|
||||
"category": "دستهبندی",
|
||||
"collection": "کالکشن",
|
||||
"howToUse": "نحوه استفاده",
|
||||
"labTests": "تستها",
|
||||
"shade": "شید",
|
||||
"colorCode": "کد رنگ",
|
||||
"ingredients": "ترکیبات",
|
||||
"texture": "بافت",
|
||||
"spf": "SPF",
|
||||
"longevity": "ماندگاری",
|
||||
}
|
||||
|
||||
# Product-level keys (variant-only keys excluded from default product queries).
|
||||
PRODUCT_DETAIL_KEYS = [
|
||||
"brand",
|
||||
"madeIn",
|
||||
"size",
|
||||
"pao",
|
||||
"finish",
|
||||
"coverage",
|
||||
"skinType",
|
||||
"category",
|
||||
"collection",
|
||||
"howToUse",
|
||||
"labTests",
|
||||
"ingredients",
|
||||
"texture",
|
||||
"spf",
|
||||
"longevity",
|
||||
]
|
||||
|
||||
VARIANT_DETAIL_KEYS = ["shade", "colorCode", "skinTone", "skinUndertone"]
|
||||
|
||||
# SerpAPI / search query templates. `{name}` and `{brand}` are filled per product.
|
||||
DETAIL_QUERY_TEMPLATES: dict[str, str] = {
|
||||
"brand": "{name} brand official",
|
||||
"madeIn": "{name} {brand} made in country of origin",
|
||||
"size": "{name} {brand} size ml volume",
|
||||
"pao": "{name} {brand} PAO period after opening",
|
||||
"finish": "{name} {brand} finish effect",
|
||||
"coverage": "{name} {brand} coverage",
|
||||
"skinType": "{name} {brand} skin type",
|
||||
"category": "{name} {brand} product category",
|
||||
"collection": "{name} {brand} collection line",
|
||||
"howToUse": "{name} {brand} how to use apply",
|
||||
"labTests": "{name} {brand} lab tests dermatologically tested",
|
||||
"ingredients": "{name} {brand} ingredients INCI",
|
||||
"texture": "{name} {brand} texture formula",
|
||||
"spf": "{name} {brand} SPF sun protection",
|
||||
"longevity": "{name} {brand} wear time longevity hours",
|
||||
"shade": "{name} {brand} shade {shade}",
|
||||
"colorCode": "{name} {brand} shade code {shade}",
|
||||
"skinTone": "{name} {brand} {shade} skin tone",
|
||||
"skinUndertone": "{name} {brand} {shade} undertone",
|
||||
}
|
||||
|
||||
SERP_ENDPOINT = "https://serpapi.com/search.json"
|
||||
|
||||
|
||||
def detail_label(key: str) -> str:
|
||||
return DETAIL_LABELS.get(key, key)
|
||||
|
||||
|
||||
def _rows_from_value(value: Any) -> list[dict]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return [r for r in value if isinstance(r, dict)]
|
||||
if isinstance(value, dict):
|
||||
return [
|
||||
{"key": str(k), "label": detail_label(str(k)), "value": v}
|
||||
for k, v in value.items()
|
||||
if v not in (None, "")
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def normalize_details(value: Any, *, prefer_keys: list[str] | None = None) -> list[dict]:
|
||||
"""Normalize details to [{key,label,value}, ...] with stable keys/labels."""
|
||||
rows = _rows_from_value(value)
|
||||
out: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
for row in rows:
|
||||
key = str(row.get("key") or "").strip()
|
||||
label = str(row.get("label") or "").strip()
|
||||
raw_val = row.get("value")
|
||||
if raw_val is None:
|
||||
continue
|
||||
if isinstance(raw_val, list):
|
||||
text = "، ".join(str(x) for x in raw_val if x not in (None, ""))
|
||||
else:
|
||||
text = str(raw_val).strip()
|
||||
if not text:
|
||||
continue
|
||||
if not key:
|
||||
# Map known Persian labels back to keys when importers omit key.
|
||||
reverse = {v: k for k, v in DETAIL_LABELS.items()}
|
||||
key = reverse.get(label, re.sub(r"[^\w]+", "_", label.lower()).strip("_") or "field")
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append({"key": key, "label": detail_label(key) if key in DETAIL_LABELS else (label or key), "value": text})
|
||||
|
||||
# Optional: keep preferred product keys first for readable exports.
|
||||
if prefer_keys:
|
||||
rank = {k: i for i, k in enumerate(prefer_keys)}
|
||||
out.sort(key=lambda r: (rank.get(r["key"], 1000), r["key"]))
|
||||
return out
|
||||
|
||||
|
||||
def build_detail_queries(
|
||||
product: dict,
|
||||
*,
|
||||
existing: dict | None = None,
|
||||
variant: dict | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Build/merge SerpAPI query strings for product (and optional variant) detail keys."""
|
||||
name = str(product.get("name") or product.get("nameFa") or "").strip() or "KIKO product"
|
||||
brand = "KIKO Milano"
|
||||
for row in normalize_details(product.get("details")):
|
||||
if row["key"] == "brand" and row["value"]:
|
||||
brand = row["value"]
|
||||
break
|
||||
shade = ""
|
||||
if variant:
|
||||
shade = str(variant.get("shade") or variant.get("colorCode") or "").strip()
|
||||
|
||||
keys = list(PRODUCT_DETAIL_KEYS)
|
||||
if variant:
|
||||
keys = list(VARIANT_DETAIL_KEYS)
|
||||
|
||||
merged = dict(existing or {})
|
||||
for key in keys:
|
||||
if merged.get(key):
|
||||
continue
|
||||
template = DETAIL_QUERY_TEMPLATES.get(key)
|
||||
if not template:
|
||||
continue
|
||||
merged[key] = (
|
||||
template.replace("{name}", name)
|
||||
.replace("{brand}", brand)
|
||||
.replace("{shade}", shade or name)
|
||||
.strip()
|
||||
)
|
||||
return merged
|
||||
|
||||
|
||||
def ensure_product_queries(product: dict) -> dict:
|
||||
"""Mutate product: normalize details + attach detailQueries for import/export."""
|
||||
product["details"] = normalize_details(
|
||||
product.get("details") or product.get("specifications"),
|
||||
prefer_keys=PRODUCT_DETAIL_KEYS,
|
||||
)
|
||||
product.pop("specifications", None)
|
||||
product["detailQueries"] = build_detail_queries(
|
||||
product, existing=product.get("detailQueries") if isinstance(product.get("detailQueries"), dict) else None
|
||||
)
|
||||
|
||||
variants = product.get("colorVariants") or product.get("variants") or []
|
||||
if isinstance(variants, list):
|
||||
for v in variants:
|
||||
if not isinstance(v, dict):
|
||||
continue
|
||||
v["details"] = normalize_details(
|
||||
v.get("details") or v.get("specifications"),
|
||||
prefer_keys=VARIANT_DETAIL_KEYS,
|
||||
)
|
||||
v.pop("specifications", None)
|
||||
v["detailQueries"] = build_detail_queries(
|
||||
product,
|
||||
existing=v.get("detailQueries") if isinstance(v.get("detailQueries"), dict) else None,
|
||||
variant=v,
|
||||
)
|
||||
product["colorVariants"] = variants
|
||||
product.pop("variants", None)
|
||||
return product
|
||||
|
||||
|
||||
def missing_detail_keys(product: dict, *, overwrite: bool = False) -> list[str]:
|
||||
present = {
|
||||
row["key"]
|
||||
for row in normalize_details(product.get("details"))
|
||||
if (overwrite is False and (row.get("value") or "").strip())
|
||||
}
|
||||
if overwrite:
|
||||
present = set()
|
||||
return [k for k in PRODUCT_DETAIL_KEYS if k not in present and k in DETAIL_QUERY_TEMPLATES]
|
||||
|
||||
|
||||
def serp_search(query: str, *, api_key: str, num: int = 5, timeout: int = 45) -> dict:
|
||||
"""Run a Google search via SerpAPI. Returns parsed JSON."""
|
||||
query = (query or "").strip()
|
||||
api_key = (api_key or "").strip()
|
||||
if not query:
|
||||
raise ValueError("Empty SerpAPI query")
|
||||
if not api_key:
|
||||
raise ValueError("SerpAPI key is not configured")
|
||||
params = {
|
||||
"engine": "google",
|
||||
"q": query,
|
||||
"api_key": api_key,
|
||||
"num": str(max(1, min(int(num), 10))),
|
||||
"hl": "en",
|
||||
"gl": "gb",
|
||||
}
|
||||
url = SERP_ENDPOINT + "?" + urllib.parse.urlencode(params)
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"User-Agent": "kikomiko-admin/1.0", "Accept": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8", "ignore")
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", "ignore")[:300]
|
||||
raise ValueError(f"SerpAPI HTTP {exc.code}: {detail or exc.reason}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise ValueError(f"SerpAPI unreachable: {getattr(exc, 'reason', exc)}") from exc
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError("SerpAPI returned invalid JSON") from exc
|
||||
|
||||
|
||||
def extract_answer_from_serp(payload: dict) -> str:
|
||||
"""Pick the best short answer text from a SerpAPI response."""
|
||||
if not isinstance(payload, dict):
|
||||
return ""
|
||||
answer = payload.get("answer_box") or {}
|
||||
if isinstance(answer, dict):
|
||||
for key in ("answer", "snippet", "result"):
|
||||
if answer.get(key):
|
||||
return str(answer[key]).strip()
|
||||
if answer.get("snippet_highlighted_words"):
|
||||
return " ".join(str(x) for x in answer["snippet_highlighted_words"])
|
||||
|
||||
kg = payload.get("knowledge_graph") or {}
|
||||
if isinstance(kg, dict):
|
||||
for key in ("description", "title"):
|
||||
if kg.get(key):
|
||||
return str(kg[key]).strip()
|
||||
|
||||
for item in payload.get("organic_results") or []:
|
||||
if isinstance(item, dict) and item.get("snippet"):
|
||||
return str(item["snippet"]).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def fill_details_via_serp(
|
||||
product: dict,
|
||||
*,
|
||||
api_key: str,
|
||||
keys: list[str] | None = None,
|
||||
overwrite: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Fill missing product details using detailQueries + SerpAPI. Mutates product."""
|
||||
ensure_product_queries(product)
|
||||
targets = keys or missing_detail_keys(product, overwrite=overwrite)
|
||||
queries = product.get("detailQueries") or {}
|
||||
by_key = {row["key"]: row for row in product.get("details") or []}
|
||||
filled: list[str] = []
|
||||
errors: list[str] = []
|
||||
for key in targets:
|
||||
query = (queries.get(key) or DETAIL_QUERY_TEMPLATES.get(key) or "").strip()
|
||||
if not query:
|
||||
continue
|
||||
# Resolve templates if still templated
|
||||
query = (
|
||||
query.replace("{name}", str(product.get("name") or ""))
|
||||
.replace("{brand}", "KIKO Milano")
|
||||
.replace("{shade}", "")
|
||||
.strip()
|
||||
)
|
||||
try:
|
||||
payload = serp_search(query, api_key=api_key)
|
||||
answer = extract_answer_from_serp(payload)
|
||||
if not answer:
|
||||
errors.append(f"{key}: no SerpAPI answer")
|
||||
continue
|
||||
by_key[key] = {"key": key, "label": detail_label(key), "value": answer}
|
||||
filled.append(key)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append(f"{key}: {exc}")
|
||||
if "HTTP 401" in str(exc) or "HTTP 403" in str(exc) or "HTTP 429" in str(exc):
|
||||
break
|
||||
# Keep preferred order
|
||||
ordered = []
|
||||
seen = set()
|
||||
for key in PRODUCT_DETAIL_KEYS:
|
||||
if key in by_key:
|
||||
ordered.append(by_key[key])
|
||||
seen.add(key)
|
||||
for key, row in by_key.items():
|
||||
if key not in seen:
|
||||
ordered.append(row)
|
||||
product["details"] = ordered
|
||||
return {
|
||||
"id": product.get("id"),
|
||||
"slug": product.get("slug"),
|
||||
"filled": filled,
|
||||
"errors": errors,
|
||||
"targets": targets,
|
||||
}
|
||||
|
||||
|
||||
def default_catalog_detail_queries() -> dict[str, str]:
|
||||
"""Catalog-level template map exported for tooling / future importers."""
|
||||
return dict(DETAIL_QUERY_TEMPLATES)
|
||||
Reference in New Issue
Block a user