"""Rebuild saved held-out predictions; this script is not run by the static Space."""
from pathlib import Path
import hashlib
import json
import math
import platform
import shutil
import numpy as np
import pandas as pd
import sklearn
from sklearn.compose import ColumnTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import Ridge
from sklearn.metrics import mean_absolute_error, r2_score
from sklearn.model_selection import GroupShuffleSplit
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import OneHotEncoder

ROOT = Path(__file__).resolve().parent
SOURCE = ROOT / 'pilot_candidates.csv'
if not SOURCE.exists():
    shutil.copyfile(ROOT.parent / 'data/off_pilot_2026_09_29/english_core_candidates.csv', SOURCE)
df = pd.read_csv(SOURCE, dtype={'code': str}).fillna('')
original_count = len(df)
df['sugars_100g'] = pd.to_numeric(df['sugars_100g'], errors='coerce')
# Package mass is a conservative proxy, NOT verification of nutrition basis.
df = df.loc[(df.product_quantity_unit.str.lower() == 'g') &
            df.sugars_100g.between(0, 100) &
            df.ingredients_text_en.str.strip().ne('') &
            df.comparison_category.str.strip().ne('') &
            df.ingredient_group_sha256.str.strip().ne('')].copy()
eligible_count = len(df)
# One observation per exact ingredient group avoids repeated calibration units.
df = df.sort_values('code').drop_duplicates('ingredient_group_sha256').reset_index(drop=True)
groups = df.ingredient_group_sha256
train_cal, test = next(GroupShuffleSplit(n_splits=1, test_size=.2, random_state=42).split(df, groups=groups))
tr, ca = next(GroupShuffleSplit(n_splits=1, test_size=.25, random_state=43).split(df.iloc[train_cal], groups=groups.iloc[train_cal]))
train, cal = train_cal[tr], train_cal[ca]
assert not (set(groups.iloc[train]) & set(groups.iloc[test]))
assert not (set(groups.iloc[cal]) & (set(groups.iloc[train]) | set(groups.iloc[test])))
parts = {'train': train, 'calibration': cal, 'test': test}
y = df.sugars_100g.to_numpy(float)
medians = df.iloc[train].groupby('comparison_category').sugars_100g.median()
fallback = float(np.median(y[train]))
baseline = df.comparison_category.map(medians).fillna(fallback).to_numpy(float)
model = make_pipeline(ColumnTransformer([
    ('ingredients', TfidfVectorizer(ngram_range=(1, 2), min_df=1, max_features=5000), 'ingredients_text_en'),
    ('category', OneHotEncoder(handle_unknown='ignore'), ['comparison_category']),
]), Ridge(alpha=10, solver='lsqr'))
model.fit(df.iloc[train], y[train])
pred = np.clip(model.predict(df), 0, 100)
results = {}
intervals = {}
for key, p in [('category', baseline), ('ingredients', pred)]:
    residuals = np.abs(y[cal] - p[cal])
    rank = math.ceil((len(cal) + 1) * .9)
    if rank > len(cal):
        raise ValueError('Too few calibration examples for a finite 90% interval')
    q = float(np.sort(residuals)[rank - 1])
    lo, hi = np.maximum(0, p[test] - q), np.minimum(100, p[test] + q)
    intervals[key] = (lo, hi)
    modified = y[test] / 10
    changed = ~np.isclose(modified, y[test])
    flagged = (y[test] < lo) | (y[test] > hi)
    synthetic_flagged = (modified < lo) | (modified > hi)
    results[key] = {
        'mae': float(mean_absolute_error(y[test], p[test])),
        'r2': float(r2_score(y[test], p[test])),
        'coverage': float(np.mean(~flagged)),
        'mean_width': float(np.mean(hi-lo)),
        'calibration_radius': q,
        'untouched_flag_count': int(flagged.sum()),
        'synthetic_changed_count': int(changed.sum()),
        'synthetic_flag_count': int(synthetic_flagged[changed].sum()),
        'synthetic_flag_rate': float(synthetic_flagged[changed].mean()) if changed.any() else None,
    }
examples = []
for j, i in enumerate(test):
    row = df.iloc[i]
    item = {'code': row.code, 'name': row.product_name or row.code,
            'category_name': row.comparison_category, 'ingredient_text': row.ingredients_text_en,
            'sugar': float(y[i]), 'url': row.product_url,
            'category_seen': bool(row.comparison_category in medians.index)}
    for key, p in [('category', baseline), ('ingredients', pred)]:
        lo, hi = intervals[key]
        item[key] = {'prediction': float(p[i]), 'low': float(lo[j]), 'high': float(hi[j])}
    examples.append(item)
data = {'source_count': original_count, 'eligible_count': eligible_count,
        'unique_count': len(df), 'counts': {k: len(v) for k, v in parts.items()},
        'source_sha256': hashlib.sha256(SOURCE.read_bytes()).hexdigest(),
        'versions': {'python': platform.python_version(), 'sklearn': sklearn.__version__, 'pandas': pd.__version__, 'numpy': np.__version__},
        'nominal_coverage': .9, 'results': results, 'examples': examples,
        'unseen_test_categories': int(sum(not x['category_seen'] for x in examples))}
out = json.dumps(data, indent=2, ensure_ascii=False, allow_nan=False)
(ROOT / 'results.json').write_text(out, encoding='utf-8')
(ROOT / 'demo-data.js').write_text('window.DEMO_DATA = ' + out.replace('<', '\\u003c') + ';\n', encoding='utf-8')
df['split'] = ''
for name, indices in parts.items():
    df.loc[indices, 'split'] = name
df[['code', 'ingredient_group_sha256', 'split']].to_csv(ROOT / 'split_manifest.csv', index=False)
print(json.dumps({k: v for k, v in data.items() if k != 'examples'}, indent=2))