import { createHash } from 'node:crypto' import type { BidOptimizationResult, BidRiskParams, ForecastBundle, ForecastKind } from '@vpp/domain' import { Decimal, LedgerService } from '@vpp/services' import type { SkillClient } from './client.js' import type { Dataset, DatasetDay } from './dataset.js' import { nums, toCurve } from './dataset.js' import { coverage, directionAccuracy, hindsightBid, mape, mean, naiveBid, nrmse, realisedRevenue, } from './metrics.js' /** * L2 harness (docs/12 §1 L2, §5 harness/): rolling-origin backtest over a * replay dataset. For each held-out day it asks the skill service for the three * forecasts and a bid, scores them against actuals, and pushes the bid through * the real LedgerService so "MILP output respects ledger constraints" is * checked by the component that enforces it — not by a re-implementation. * * Every run produces an EvalRun whose `digest` covers config, dataset hash, * skill versions and metrics. Two runs on the same inputs must digest equal * (reproducibility acceptance); the CLI's --check compares against the * committed baseline. */ export interface HarnessConfig { /** History days handed to each forecast call. */ window: number /** Index of the first held-out day (needs ≥ window history before it). */ holdoutFrom: number /** Number of held-out days; 0 = to the end of the dataset. */ holdoutDays: number risk: BidRiskParams /** * Share of sellable energy (k·cap·0.25·96) used as the daily contract * position when seeding the ledger. Synthetic-set convenience, not a market * parameter. */ contractShareOfSellable: string /** Passed to LedgerService; OPEN-QUESTION A5 — placeholder until confirmed. */ daMonthlyDeviationBand: string } export const DEFAULT_CONFIG: HarnessConfig = { window: 28, holdoutFrom: 60, holdoutDays: 0, // OPEN-QUESTION B6 (commitment_buffer_k) — placeholder for harness use only. // OPEN-QUESTION A6 (marginal_cost) — placeholder 0: offer as a price-taker. risk: { risk_aversion: '0.3', commitment_buffer_k: '0.9', min_block_mwh: '0.5', marginal_cost_yuan_per_mwh: '0' }, contractShareOfSellable: '0.6', // OPEN-QUESTION A5 — placeholder, same value the services ledger tests use. daMonthlyDeviationBand: '0.05', } export interface ForecastMetrics { days: number mape: number | null nrmse: number | null coverage_p10_p90: number direction_accuracy: number | null } export interface BidMetrics { days: number optimal_days: number ledger_accepted_days: number revenue_skill_yuan: number revenue_naive_yuan: number revenue_hindsight_yuan: number /** skill / hindsight — 1.0 would be perfect foresight. */ capture_ratio: number /** skill / naive — > 1.0 means the optimiser beats a price-taker. */ uplift_vs_naive: number } export interface DayResult { date: string load: { mape: number; coverage: number } pv: { nrmse: number; coverage: number } price: { mape: number; coverage: number; direction: number } bid: { status: string ledger_accepted: boolean energy_mwh: string revenue_skill: number revenue_naive: number revenue_hindsight: number } } export interface EvalRun { id: string layer: 'L2' run_at: string dataset: { name: string; sha256: string; synthetic: boolean } skill_versions: Record config: HarnessConfig metrics: { load: ForecastMetrics; pv: ForecastMetrics; price: ForecastMetrics; bid: BidMetrics } per_day: DayResult[] /** sha256 over everything above except id/run_at — the reproducibility key. */ digest: string } const REF = 'f'.repeat(64) /** Key-sorted JSON. (services' canonicalJson rejects floats by design; metrics are floats.) */ export const stableJson = (v: unknown): string => JSON.stringify(v, (_k, val) => val !== null && typeof val === 'object' && !Array.isArray(val) ? Object.fromEntries(Object.entries(val as Record).sort(([a], [b]) => (a < b ? -1 : 1))) : val, ) const FIELD: Record = { LOAD: 'load_mw', PV: 'pv_mw', PRICE: 'price_yuan_per_mwh', } const q = (x: number, dp: number) => Number(x.toFixed(dp)) function bandOf(bundle: ForecastBundle) { return { p10: nums(bundle.quantiles.p10.values), p50: nums(bundle.quantiles.p50.values), p90: nums(bundle.quantiles.p90.values), } } function seedLedger(dataset: Dataset, cfg: HarnessConfig, days: DatasetDay[]): LedgerService { const ledger = new LedgerService({ daMonthlyDeviationBand: cfg.daMonthlyDeviationBand, clock: () => '2026-01-01T00:00:00Z' }) const k = new Decimal(cfg.risk.commitment_buffer_k) const share = new Decimal(cfg.contractShareOfSellable) const months = [...new Set(days.map((d) => d.date.slice(0, 7)))] let version = 0 for (const month of months) { const sample = days.find((d) => d.date.startsWith(month))! const sellable = sample.adjustable_capacity_mw .reduce((s, v) => s.add(new Decimal(v)), new Decimal(0)) .mul(k) .mul('0.25') const daysInMonth = new Date(Date.UTC(Number(month.slice(0, 4)), Number(month.slice(5, 7)), 0)).getUTCDate() ledger.append({ id: `contract-${month}`, timescale: 'MONTHLY', period: month, kind: 'CONTRACT', energy_mwh: sellable.mul(share).mul(daysInMonth).toFixed(3), curve: null, source_ref: `synthetic-contract-${month}`, expected_version: version++, }) } void dataset return ledger } export async function runL2( dataset: Dataset, datasetSha256: string, client: SkillClient, cfg: HarnessConfig = DEFAULT_CONFIG, clock: () => string = () => new Date().toISOString(), ): Promise { if (cfg.holdoutFrom < cfg.window) throw new Error('holdoutFrom must be ≥ window') const end = cfg.holdoutDays > 0 ? Math.min(dataset.days.length, cfg.holdoutFrom + cfg.holdoutDays) : dataset.days.length const holdout = dataset.days.slice(cfg.holdoutFrom, end) const ledger = seedLedger(dataset, cfg, holdout) const skillVersions = Object.fromEntries((await client.skills()).map((s) => [s.id, s.version])) const perDay: DayResult[] = [] for (let idx = cfg.holdoutFrom; idx < end; idx++) { const day = dataset.days[idx]! const history = dataset.days.slice(idx - cfg.window, idx) const forecasts = {} as Record for (const kind of ['LOAD', 'PV', 'PRICE'] as const) { forecasts[kind] = await client.forecast(kind, { kind, market_date: day.date, unit: kind === 'PRICE' ? 'yuan_per_mwh' : 'mw', history: history.map((h) => toCurve(h[FIELD[kind]] as string[], h.date)), exogenous: {}, features_snapshot_ref: REF, }) } const bounds = ledger.dayAheadBounds(day.date) const bid: BidOptimizationResult = await client.optimizeBid({ market_date: day.date, price_forecast: forecasts.PRICE, adjustable_capacity_mw: toCurve(day.adjustable_capacity_mw, day.date), position_bounds: bounds, risk: cfg.risk, }) let ledgerAccepted = false if (bid.solver.status === 'OPTIMAL') { try { ledger.append({ id: `bid-${day.date}`, timescale: 'DAY_AHEAD', period: day.date, kind: 'BID_SUBMITTED', energy_mwh: bid.daily_energy_mwh, curve: bid.quantities_mwh, source_ref: `eval-${day.date}`, expected_version: ledger.read().version, }) ledgerAccepted = true } catch { ledgerAccepted = false } } const actualLoad = nums(day.load_mw) const actualPv = nums(day.pv_mw) const actualPrice = nums(day.price_yuan_per_mwh) const load = bandOf(forecasts.LOAD) const pv = bandOf(forecasts.PV) const price = bandOf(forecasts.PRICE) const capMwh = nums(day.adjustable_capacity_mw).map((c) => c * Number(cfg.risk.commitment_buffer_k) * 0.25) const eMax = Number(bounds.daily_energy_max_mwh) const daylight = actualPv.map((v, i) => [v, i] as const).filter(([v]) => v > 0).map(([, i]) => i) perDay.push({ date: day.date, load: { mape: q(mape(actualLoad, load.p50), 6), coverage: q(coverage(actualLoad, load.p10, load.p90), 6) }, pv: { nrmse: q(nrmse(actualPv, pv.p50, Number(dataset.meta.pv_capacity_mw)), 6), coverage: q( coverage( daylight.map((i) => actualPv[i]!), daylight.map((i) => pv.p10[i]!), daylight.map((i) => pv.p90[i]!), ), 6, ), }, price: { mape: q(mape(actualPrice, price.p50), 6), coverage: q(coverage(actualPrice, price.p10, price.p90), 6), direction: q(directionAccuracy(actualPrice, price.p50), 6), }, bid: { status: bid.solver.status, ledger_accepted: ledgerAccepted, energy_mwh: bid.daily_energy_mwh, revenue_skill: q( realisedRevenue({ offers: nums(bid.prices_yuan_per_mwh.values), quantities: nums(bid.quantities_mwh.values) }, actualPrice), 2, ), revenue_naive: q(realisedRevenue(naiveBid(capMwh, eMax), actualPrice), 2), revenue_hindsight: q( realisedRevenue(hindsightBid(actualPrice, capMwh, eMax, Number(cfg.risk.min_block_mwh)), actualPrice), 2, ), }, }) } const fm = (pick: (d: DayResult) => { mape?: number; nrmse?: number; coverage: number; direction?: number }): ForecastMetrics => { const rows = perDay.map(pick) const has = (k: 'mape' | 'nrmse' | 'direction') => rows.every((r) => r[k] !== undefined) return { days: rows.length, mape: has('mape') ? q(mean(rows.map((r) => r.mape!)), 6) : null, nrmse: has('nrmse') ? q(mean(rows.map((r) => r.nrmse!)), 6) : null, coverage_p10_p90: q(mean(rows.map((r) => r.coverage)), 6), direction_accuracy: has('direction') ? q(mean(rows.map((r) => r.direction!)), 6) : null, } } const sum = (f: (d: DayResult) => number) => q(perDay.reduce((s, d) => s + f(d), 0), 2) const skill = sum((d) => d.bid.revenue_skill) const naive = sum((d) => d.bid.revenue_naive) const hindsight = sum((d) => d.bid.revenue_hindsight) const body = { layer: 'L2' as const, dataset: { name: dataset.meta.name, sha256: datasetSha256, synthetic: dataset.meta.synthetic }, skill_versions: skillVersions, config: cfg, metrics: { load: fm((d) => d.load), pv: fm((d) => d.pv), price: fm((d) => d.price), bid: { days: perDay.length, optimal_days: perDay.filter((d) => d.bid.status === 'OPTIMAL').length, ledger_accepted_days: perDay.filter((d) => d.bid.ledger_accepted).length, revenue_skill_yuan: skill, revenue_naive_yuan: naive, revenue_hindsight_yuan: hindsight, capture_ratio: hindsight === 0 ? 0 : q(skill / hindsight, 6), uplift_vs_naive: naive === 0 ? 0 : q(skill / naive, 6), }, }, per_day: perDay, } const digest = createHash('sha256').update(stableJson(body)).digest('hex') const runAt = clock() return { id: `l2-${runAt.replace(/[:.]/g, '-')}-${digest.slice(0, 8)}`, run_at: runAt, ...body, digest } }