build_web_data.py

#!/usr/bin/env python3
"""
Build optimized web_data.json from glovo_batumi_full.json.
Extracts only fields needed for the search UI, outputs compact JSON.
"""

import json
import os
import sys

INPUT = os.path.join(os.path.dirname(__file__), "glovo_batumi_full.json")
OUTPUT = os.path.join(os.path.dirname(__file__), "web_data.json")

FIELDS = ["name", "price", "promo_price", "store_name", "glovo_category", "category"]

# Map glovo_category to short keys to save space
CAT_MAP = {
    "food_1": "F",
    "groceries_4": "G",
    "shops_22": "S",
    "pharmacy-beauty_3": "P",
}


def main():
    print(f"Loading {INPUT}...")
    with open(INPUT, "r", encoding="utf-8") as f:
        data = json.load(f)

    print(f"  Loaded {len(data)} items")

    # Build compact records: [name, price, promo_price, store_name, glovo_cat_short, category]
    # Use array format instead of objects to save ~40% space
    records = []
    for item in data:
        name = item.get("name", "")
        price = item.get("price")
        promo = item.get("promo_price")
        store = item.get("store_name", "")
        gcat = CAT_MAP.get(item.get("glovo_category", ""), "?")
        cat = item.get("category") or ""

        # Round prices to 2 decimals
        if price is not None:
            price = round(price, 2)
        if promo is not None:
            promo = round(promo, 2)

        records.append([name, price, promo, store, gcat, cat])

    # Collect unique store names and categories for deduplication
    stores = sorted(set(r[3] for r in records))
    cats = sorted(set(r[5] for r in records))

    store_idx = {s: i for i, s in enumerate(stores)}
    cat_idx = {c: i for i, c in enumerate(cats)}

    # Replace store/cat strings with indices
    compact = []
    for r in records:
        compact.append(
            [
                r[0],  # name
                r[1],  # price
                r[2],  # promo_price (null ok)
                store_idx[r[3]],  # store index
                r[4],  # glovo_cat short
                cat_idx[r[5]],  # category index
            ]
        )

    output = {
        "stores": stores,
        "cats": cats,
        "items": compact,
    }

    print(f"Writing {OUTPUT}...")
    with open(OUTPUT, "w", encoding="utf-8") as f:
        json.dump(output, f, ensure_ascii=False, separators=(",", ":"))

    size = os.path.getsize(OUTPUT)
    print(f"  Done: {len(compact)} items, {size:,} bytes ({size / 1024 / 1024:.1f} MB)")
    print(f"  Stores: {len(stores)}, Categories: {len(cats)}")


if __name__ == "__main__":
    main()