Files
Kikomiko/kikomiko/admin/kiko_enrich.py
T

626 lines
21 KiB
Python

"""Fetch one KIKO product page and map it to Kikomiko importer fields.
Used by the admin "دریافت از KIKO" button so each product can get its own
description + details table without creating a duplicate family.
"""
from __future__ import annotations
import json
import re
import ssl
import urllib.error
import urllib.parse
import urllib.request
from typing import Any
def _ssl_context() -> ssl.SSLContext:
"""Prefer system/certifi CAs; fall back if the host has a broken cert chain."""
try:
import certifi # type: ignore
return ssl.create_default_context(cafile=certifi.where())
except Exception:
try:
return ssl.create_default_context()
except Exception:
return ssl._create_unverified_context()
UA = (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
try:
from detail_schema import DETAIL_LABELS, ensure_product_queries
except ImportError: # pragma: no cover
DETAIL_LABELS = {
"brand": "برند",
"madeIn": "ساخت",
"size": "حجم / سایز",
"pao": "PAO",
"finish": "فینیش",
"coverage": "پوشش",
"skinType": "نوع پوست",
"skinTone": "ته‌رنگ پوست",
"skinUndertone": "آندرتون",
"category": "دسته‌بندی",
"collection": "کالکشن",
"howToUse": "نحوه استفاده",
"labTests": "تست‌ها",
"shade": "شید",
"colorCode": "کد رنگ",
}
def ensure_product_queries(product: dict) -> dict:
return product
COUNTRY = {
"IT": "ایتالیا",
"FR": "فرانسه",
"US": "آمریکا",
"GB": "بریتانیا",
"DE": "آلمان",
"ES": "اسپانیا",
}
DEFAULT_MARKET = "en-gb"
def _http_get(url: str, timeout: int = 45) -> str:
req = urllib.request.Request(
url,
headers={
"User-Agent": UA,
"Accept": "text/html,application/xhtml+xml",
"Accept-Language": "en-GB,en;q=0.9",
},
)
last_err: Exception | None = None
for ctx in (_ssl_context(), ssl._create_unverified_context()):
try:
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
return resp.read().decode("utf-8", "ignore")
except urllib.error.HTTPError as exc:
raise ValueError(f"KIKO HTTP {exc.code} for {url}") from exc
except urllib.error.URLError as exc:
last_err = exc
continue
raise ValueError(f"Could not reach KIKO: {getattr(last_err, 'reason', last_err)}")
def _next_data(html: str) -> dict:
m = re.search(
r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>',
html,
re.S | re.I,
)
if not m:
raise ValueError("KIKO page has no __NEXT_DATA__ (not a product page, or blocked).")
return json.loads(m.group(1))
def _slugify(value: str) -> str:
text = re.sub(r"[^\w\s-]", "", value or "", flags=re.U)
text = re.sub(r"[\s_]+", "-", text.strip().lower())
return re.sub(r"-+", "-", text).strip("-")
def _detail(key: str, value: Any) -> dict | None:
if value is None:
return None
if isinstance(value, list):
text = "، ".join(str(x) for x in value if x not in (None, ""))
else:
text = str(value).strip()
if not text:
return None
return {"key": key, "label": DETAIL_LABELS.get(key, key), "value": text}
def _parse_shade(color: str | None) -> dict:
raw = str(color or "").strip()
m = re.match(r"^(\d+(?:\.\d+)?)([A-Z]{0,3})\s*(.*)$", raw, re.I)
if not m:
return {"colorCode": None, "colorName": raw or None, "shade": raw or None}
code = f"{m.group(1)}{(m.group(2) or '').upper()}"
name = (m.group(3) or "").strip() or None
return {
"colorCode": code,
"colorName": name,
"shade": f"{code} {name}" if name else code,
}
def _media_urls(item: dict) -> list[str]:
urls: list[str] = []
def push(u: Any) -> None:
if isinstance(u, str) and u and u not in urls:
urls.append(u)
primary = item.get("primary_image")
if isinstance(primary, dict):
push(primary.get("url"))
product_media = item.get("product_media") or {}
if isinstance(product_media, dict):
for m in product_media.get("media") or []:
if isinstance(m, dict):
push(m.get("url"))
else:
push(m)
for u in item.get("media_urls") or []:
push(u)
for m in item.get("extra_media") or []:
if isinstance(m, dict):
push(m.get("url"))
else:
push(m)
return urls
def _money(value: Any) -> float:
try:
n = float(value)
except (TypeError, ValueError):
return 0.0
return n if n == n else 0.0 # NaN guard
def _to_variant(item: dict, parent_slug: str, description: str = "") -> dict:
shade = _parse_shade(item.get("color"))
images = _media_urls(item)
name = (
item.get("display_value")
or item.get("product_name")
or f"{parent_slug} {shade['shade'] or ''}".strip()
)
slug = item.get("slug") or _slugify(name)
return {
"id": str(item.get("product_id") or item.get("backend_id") or slug),
"slug": slug,
"name": name,
"shade": shade["shade"],
"colorCode": shade["colorCode"],
"colorName": shade["colorName"],
"colorHex": item.get("hex_color"),
"price": _money(item.get("display_price")),
"priceEur": _money(item.get("display_price")),
"compareAtPrice": _money(item.get("historical_price")) or None,
"stock": 1 if str(item.get("is_sellable")).lower() == "true" else 0,
"sku": item.get("backend_id") or item.get("custom_id"),
"externalId": str(item.get("product_id") or ""),
"heroImage": images[0] if images else None,
"images": images,
"description": description
or item.get("long_description")
or item.get("short_description")
or "",
"details": [
x
for x in (
_detail("shade", shade["shade"]),
_detail("colorCode", shade["colorCode"]),
_detail("skinTone", item.get("skin_tone")),
_detail("skinUndertone", item.get("skin_undertone")),
)
if x
],
"sourceAvailable": str(item.get("is_sellable")).lower() == "true",
"sourcePrice": _money(item.get("display_price")),
"sourceCurrency": item.get("currency_id"),
}
def _build_details(selected: dict, root: dict) -> list[dict]:
made_in = COUNTRY.get(selected.get("country_of_origin") or "") or selected.get(
"country_of_origin"
)
category = " / ".join(
filter(
None,
[
selected.get("product_class_lev1_desc"),
selected.get("product_class_lev2_desc"),
selected.get("product_class_lev3_desc"),
selected.get("product_class_lev4_desc"),
],
)
)
rows = [
_detail("brand", "KIKO Milano"),
_detail("madeIn", made_in),
_detail(
"size",
f"{selected['legal_volume']} ml"
if selected.get("legal_volume") is not None
else None,
),
_detail(
"pao",
f"{selected['pao']} ماه" if selected.get("pao") is not None else None,
),
_detail("finish", selected.get("finish_effect")),
_detail("coverage", selected.get("coverage")),
_detail("skinType", selected.get("specificity_for_skin_type")),
_detail("category", category),
_detail("collection", selected.get("collection")),
_detail(
"howToUse",
selected.get("ed_cont_how_to_use") or root.get("ed_cont_how_to_use"),
),
_detail(
"labTests",
selected.get("ed_cont_lab_tests") or root.get("ed_cont_lab_tests"),
),
]
return [r for r in rows if r]
def extract_from_html(html: str, source_url: str) -> dict:
data = _next_data(html)
props = data.get("props", {}).get("pageProps") or {}
root = props.get("root")
selected = props.get("selected")
children = props.get("children") or []
if not isinstance(root, dict) or not isinstance(selected, dict):
raise ValueError("Not a KIKO product detail page.")
parent_name = root.get("display_value") or root.get("product_name") or selected.get(
"display_value"
)
parent_slug = root.get("slug") or _slugify(parent_name or "kiko-product")
description = (
selected.get("long_description")
or root.get("long_description")
or selected.get("short_description")
or root.get("short_description")
or ""
)
tagline = selected.get("short_description") or root.get("short_description") or ""
by_id: dict[str, dict] = {}
for child in children:
if isinstance(child, dict):
v = _to_variant(child, parent_slug)
by_id[v["id"]] = v
selected_variant = _to_variant(selected, parent_slug, description=description)
by_id[selected_variant["id"]] = {
**by_id.get(selected_variant["id"], {}),
**selected_variant,
"images": selected_variant["images"]
or by_id.get(selected_variant["id"], {}).get("images")
or [],
}
color_variants = sorted(
by_id.values(),
key=lambda v: str(v.get("shade") or v.get("name") or ""),
)
category = " / ".join(
filter(
None,
[
selected.get("product_class_lev1_desc"),
selected.get("product_class_lev2_desc"),
selected.get("product_class_lev3_desc"),
],
)
)
sku = root.get("backend_id") or selected.get("backend_id")
external_id = str(root.get("product_id") or selected.get("product_id") or "")
product = {
"slug": parent_slug,
"name": parent_name,
"nameFa": parent_name,
"displayName": parent_name,
"category": category or None,
"description": description,
"descriptionIntro": tagline,
"tagline": tagline,
"seoTitle": parent_name,
"seoDescription": (tagline or description)[:160],
"sourceDescription": description,
"sourceTagline": tagline,
"sourceIngredients": selected.get("ingredient_list") or "",
"details": _build_details(selected, root),
"defaultVariantId": selected_variant["id"],
"heroImage": selected_variant.get("heroImage"),
"images": selected_variant.get("images") or [],
"price": selected_variant.get("price") or 0,
"priceEur": selected_variant.get("priceEur")
or selected_variant.get("sourcePrice")
or selected_variant.get("price")
or 0,
"sourcePrice": selected_variant.get("sourcePrice")
or selected_variant.get("price")
or 0,
"sourceCurrency": selected_variant.get("sourceCurrency") or "EUR",
"stock": sum(int(v.get("stock") or 0) for v in color_variants),
"sku": sku,
"externalId": external_id,
"source": "kikocosmetics",
"sourceUrl": source_url.split("?")[0],
"colorVariants": color_variants,
"match": {
"sku": sku,
"externalId": external_id,
"slugs": list(
dict.fromkeys(
[parent_slug] + [v.get("slug") for v in color_variants if v.get("slug")]
)
),
"name": parent_name,
},
}
return ensure_product_queries(product)
def normalize_kiko_url(url: str, market: str = DEFAULT_MARKET) -> str:
url = (url or "").strip()
if not url:
raise ValueError("KIKO product URL is required.")
# Allow pasting host-only / path-only values.
if url in ("kikocosmetics.com", "www.kikocosmetics.com"):
raise ValueError(
"Paste a full product page URL, e.g. "
"https://www.kikocosmetics.com/en-gb/p/love-fusion-24h-moisture-radiant-foundation-1-52012/"
)
if not url.startswith("http"):
if url.startswith("kikocosmetics.com") or url.startswith("www.kikocosmetics.com"):
url = "https://" + url
else:
url = "https://www.kikocosmetics.com" + (
url if url.startswith("/") else f"/{url}"
)
host = urllib.parse.urlparse(url).netloc.lower()
if "kikocosmetics.com" not in host:
raise ValueError("URL must be on kikocosmetics.com")
if "/p/" not in url:
raise ValueError(
"This is not a product page. Open the product on KIKO, copy the address bar "
"(it must contain /p/…), then paste it here."
)
return url.split("#")[0]
def discover_product_url(query: str, market: str = DEFAULT_MARKET) -> str | None:
"""Best-effort search: return first /p/ URL matching the query name."""
query = (query or "").strip()
if not query:
return None
search = (
f"https://www.kikocosmetics.com/{market}/search/"
f"?query={urllib.parse.quote(query)}"
)
html = _http_get(search)
data = _next_data(html)
blob = json.dumps(data, ensure_ascii=False)
paths = re.findall(rf"/{re.escape(market)}/p/[a-z0-9-]+-\d+", blob, flags=re.I)
# Prefer paths whose slug tokens overlap the query
tokens = [t for t in re.split(r"[^a-z0-9]+", query.lower()) if len(t) > 2]
ranked = []
for path in dict.fromkeys(paths):
score = sum(1 for t in tokens if t in path.lower())
ranked.append((score, path))
ranked.sort(key=lambda x: (-x[0], x[1]))
if not ranked:
# fallback: any /p/ in raw HTML
paths = re.findall(rf"/{re.escape(market)}/p/[a-z0-9-]+-\d+", html, flags=re.I)
if not paths:
return None
return "https://www.kikocosmetics.com" + paths[0]
best_score, best = ranked[0]
if best_score <= 0 and tokens:
# weak match — still return first search hit if any
pass
return "https://www.kikocosmetics.com" + best
def _family_key_from_product_url(url: str) -> str:
"""Collapse shade URLs into one family key.
Example:
.../p/jelly-stylo-501-46800/ → jelly-stylo
.../p/daily-protection-bb-cream-spf-30-01-42063/ → daily-protection-bb-cream-spf-30
"""
path = urllib.parse.urlparse(url).path.rstrip("/")
slug = path.split("/")[-1].lower()
m = re.match(r"^(.+)-(\d+)-(\d+)$", slug)
return m.group(1) if m else slug
def discover_catalog_product_urls(
market: str = DEFAULT_MARKET,
*,
one_per_family: bool = True,
) -> dict[str, Any]:
"""Pull product page URLs from the official KIKO market sitemap."""
market = (market or DEFAULT_MARKET).strip() or DEFAULT_MARKET
sitemap_url = f"https://www.kikocosmetics.com/{market}/sitemap.xml"
xml = _http_get(sitemap_url, timeout=90)
pattern = rf"https://www\.kikocosmetics\.com/{re.escape(market)}/p/[a-z0-9-]+/?"
all_urls = [u.rstrip("/") + "/" for u in re.findall(pattern, xml, flags=re.I)]
# preserve sitemap order, drop dupes
seen: set[str] = set()
ordered: list[str] = []
for u in all_urls:
key = u.lower()
if key in seen:
continue
seen.add(key)
ordered.append(u)
family_urls: list[str] = []
if one_per_family:
fam_seen: set[str] = set()
for u in ordered:
fam = _family_key_from_product_url(u)
if fam in fam_seen:
continue
fam_seen.add(fam)
family_urls.append(u)
else:
family_urls = list(ordered)
return {
"market": market,
"sitemapUrl": sitemap_url,
"shadeUrls": len(ordered),
"familyUrls": len(family_urls),
"urls": family_urls,
}
def fetch_kiko_product(
*,
url: str | None = None,
query: str | None = None,
market: str = DEFAULT_MARKET,
) -> dict:
market = (market or DEFAULT_MARKET).strip() or DEFAULT_MARKET
source_url = None
if url:
source_url = normalize_kiko_url(url, market=market)
elif query:
source_url = discover_product_url(query, market=market)
if not source_url:
raise ValueError(
"No KIKO product page found for that name. Paste the product URL instead."
)
else:
raise ValueError("Provide a KIKO product URL or a search name.")
html = _http_get(source_url)
product = extract_from_html(html, source_url)
product["sourceMarket"] = market
return product
def merge_enrichment(
existing: dict,
incoming: dict,
*,
overwrite_description: bool = False,
) -> dict:
"""Merge KIKO enrichment into one existing product; keep local id/slug."""
out = dict(existing)
out["id"] = existing.get("id")
out["slug"] = existing.get("slug") or incoming.get("slug")
out["details"] = incoming.get("details") or out.get("details") or []
for key in (
"sourceDescription",
"sourceTagline",
"sourceIngredients",
"sourceUrl",
"sourceMarket",
"source",
"sourcePrice",
"sourceCurrency",
"priceEur",
"sku",
"externalId",
"tagline",
"descriptionIntro",
"heroImage",
"images",
):
if incoming.get(key) not in (None, "", []):
out[key] = incoming[key]
# Keep euro retail on the product for storefront FX conversion.
if incoming.get("priceEur") not in (None, "", 0):
out["priceEur"] = incoming["priceEur"]
out["sourcePrice"] = incoming.get("sourcePrice") or incoming["priceEur"]
if incoming.get("sourceCurrency"):
out["sourceCurrency"] = incoming["sourceCurrency"]
# Do not overwrite local toman `price` with raw euro when one already exists.
existing_price = float(out.get("price") or 0)
if existing_price <= 0 or existing_price < 500:
out["price"] = incoming.get("price") or incoming["priceEur"]
incoming_desc = (incoming.get("description") or "").strip()
existing_desc = (out.get("description") or "").strip()
if overwrite_description and incoming_desc:
if existing_desc:
out["sourceDescription"] = out.get("sourceDescription") or incoming_desc
out["description"] = incoming_desc
elif not existing_desc and incoming_desc:
out["description"] = incoming_desc
elif incoming_desc:
out["sourceDescription"] = out.get("sourceDescription") or incoming_desc
# Merge variant shade details / media onto existing variants where possible.
existing_variants = list(out.get("colorVariants") or out.get("variants") or [])
by_slug = {str(v.get("slug") or ""): v for v in existing_variants if v.get("slug")}
by_code = {
str(v.get("colorCode") or ""): v for v in existing_variants if v.get("colorCode")
}
by_shade = {str(v.get("shade") or ""): v for v in existing_variants if v.get("shade")}
used = set()
merged = []
for inc in incoming.get("colorVariants") or []:
match = (
by_slug.get(str(inc.get("slug") or ""))
or by_code.get(str(inc.get("colorCode") or ""))
or by_shade.get(str(inc.get("shade") or ""))
)
if match:
item = dict(match)
for key in (
"shade",
"colorCode",
"colorName",
"colorHex",
"sku",
"externalId",
"heroImage",
"images",
"details",
"sourceAvailable",
"sourcePrice",
"sourceCurrency",
"priceEur",
):
if inc.get(key) not in (None, "", []):
item[key] = inc[key]
# Mirror euro retail onto variant price when still empty / euro-scale.
inc_eur = inc.get("priceEur") or inc.get("sourcePrice") or inc.get("price")
if inc_eur not in (None, "", 0):
item["priceEur"] = inc_eur
item["sourcePrice"] = inc.get("sourcePrice") or inc_eur
cur_price = float(item.get("price") or 0)
if cur_price <= 0 or cur_price < 500:
item["price"] = inc_eur
if not (item.get("description") or "").strip() and (inc.get("description") or "").strip():
item["description"] = inc["description"]
used.add(id(match))
merged.append(item)
else:
merged.append(dict(inc))
for v in existing_variants:
if id(v) not in used:
merged.append(v)
# Deduplicate
deduped = []
seen_ids, seen_slugs = set(), set()
for v in merged:
vid, vslug = str(v.get("id") or ""), str(v.get("slug") or "")
if vid and vid in seen_ids:
continue
if vslug and vslug in seen_slugs:
continue
if vid:
seen_ids.add(vid)
if vslug:
seen_slugs.add(vslug)
deduped.append(v)
out["colorVariants"] = deduped
out.pop("variants", None)
if incoming.get("detailQueries") and not out.get("detailQueries"):
out["detailQueries"] = incoming["detailQueries"]
return ensure_product_queries(out)