vpp-ai-platform/packages/evals/src/harness.ts

import { createHash } from 'node:crypto'
import type { BidOptimizationResult, BidRiskParams, ForecastBundle, ForecastKind } from '@vpp/domain'
import { Decimal, LedgerService } from '@vpp/services'
import type { SkillClient } from './client.js'
import type { Dataset, DatasetDay } from './dataset.js'
import { nums, toCurve } from './dataset.js'
import {
  coverage,
  directionAccuracy,
  hindsightBid,
  mape,
  mean,
  naiveBid,
  nrmse,
  realisedRevenue,
} from './metrics.js'

/**
 * L2 harness (docs/12 §1 L2, §5 harness/): rolling-origin backtest over a
 * replay dataset. For each held-out day it asks the skill service for the three
 * forecasts and a bid, scores them against actuals, and pushes the bid through
 * the real LedgerService so "MILP output respects ledger constraints" is
 * checked by the component that enforces it — not by a re-implementation.
 *
 * Every run produces an EvalRun whose `digest` covers config, dataset hash,
 * skill versions and metrics. Two runs on the same inputs must digest equal
 * (reproducibility acceptance); the CLI's --check compares against the
 * committed baseline.
 */
export interface HarnessConfig {
  /** History days handed to each forecast call. */
  window: number
  /** Index of the first held-out day (needs ≥ window history before it). */
  holdoutFrom: number
  /** Number of held-out days; 0 = to the end of the dataset. */
  holdoutDays: number
  risk: BidRiskParams
  /**
   * Share of sellable energy (k·cap·0.25·96) used as the daily contract
   * position when seeding the ledger. Synthetic-set convenience, not a market
   * parameter.
   */
  contractShareOfSellable: string
  /** Passed to LedgerService; OPEN-QUESTION A5 — placeholder until confirmed. */
  daMonthlyDeviationBand: string
}

export const DEFAULT_CONFIG: HarnessConfig = {
  window: 28,
  holdoutFrom: 60,
  holdoutDays: 0,
  // OPEN-QUESTION B6 (commitment_buffer_k) — placeholder for harness use only.
  // OPEN-QUESTION A6 (marginal_cost) — placeholder 0: offer as a price-taker.
  risk: { risk_aversion: '0.3', commitment_buffer_k: '0.9', min_block_mwh: '0.5', marginal_cost_yuan_per_mwh: '0' },
  contractShareOfSellable: '0.6',
  // OPEN-QUESTION A5 — placeholder, same value the services ledger tests use.
  daMonthlyDeviationBand: '0.05',
}

export interface ForecastMetrics {
  days: number
  mape: number | null
  nrmse: number | null
  coverage_p10_p90: number
  direction_accuracy: number | null
}

export interface BidMetrics {
  days: number
  optimal_days: number
  ledger_accepted_days: number
  revenue_skill_yuan: number
  revenue_naive_yuan: number
  revenue_hindsight_yuan: number
  /** skill / hindsight — 1.0 would be perfect foresight. */
  capture_ratio: number
  /** skill / naive — > 1.0 means the optimiser beats a price-taker. */
  uplift_vs_naive: number
}

export interface DayResult {
  date: string
  load: { mape: number; coverage: number }
  pv: { nrmse: number; coverage: number }
  price: { mape: number; coverage: number; direction: number }
  bid: {
    status: string
    ledger_accepted: boolean
    energy_mwh: string
    revenue_skill: number
    revenue_naive: number
    revenue_hindsight: number
  }
}

export interface EvalRun {
  id: string
  layer: 'L2'
  run_at: string
  dataset: { name: string; sha256: string; synthetic: boolean }
  skill_versions: Record<string, string>
  config: HarnessConfig
  metrics: { load: ForecastMetrics; pv: ForecastMetrics; price: ForecastMetrics; bid: BidMetrics }
  per_day: DayResult[]
  /** sha256 over everything above except id/run_at — the reproducibility key. */
  digest: string
}

const REF = 'f'.repeat(64)

/** Key-sorted JSON. (services' canonicalJson rejects floats by design; metrics are floats.) */
export const stableJson = (v: unknown): string =>
  JSON.stringify(v, (_k, val) =>
    val !== null && typeof val === 'object' && !Array.isArray(val)
      ? Object.fromEntries(Object.entries(val as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : 1)))
      : val,
  )
const FIELD: Record<ForecastKind, keyof DatasetDay> = {
  LOAD: 'load_mw',
  PV: 'pv_mw',
  PRICE: 'price_yuan_per_mwh',
}

const q = (x: number, dp: number) => Number(x.toFixed(dp))

function bandOf(bundle: ForecastBundle) {
  return {
    p10: nums(bundle.quantiles.p10.values),
    p50: nums(bundle.quantiles.p50.values),
    p90: nums(bundle.quantiles.p90.values),
  }
}

function seedLedger(dataset: Dataset, cfg: HarnessConfig, days: DatasetDay[]): LedgerService {
  const ledger = new LedgerService({ daMonthlyDeviationBand: cfg.daMonthlyDeviationBand, clock: () => '2026-01-01T00:00:00Z' })
  const k = new Decimal(cfg.risk.commitment_buffer_k)
  const share = new Decimal(cfg.contractShareOfSellable)
  const months = [...new Set(days.map((d) => d.date.slice(0, 7)))]
  let version = 0
  for (const month of months) {
    const sample = days.find((d) => d.date.startsWith(month))!
    const sellable = sample.adjustable_capacity_mw
      .reduce((s, v) => s.add(new Decimal(v)), new Decimal(0))
      .mul(k)
      .mul('0.25')
    const daysInMonth = new Date(Date.UTC(Number(month.slice(0, 4)), Number(month.slice(5, 7)), 0)).getUTCDate()
    ledger.append({
      id: `contract-${month}`,
      timescale: 'MONTHLY',
      period: month,
      kind: 'CONTRACT',
      energy_mwh: sellable.mul(share).mul(daysInMonth).toFixed(3),
      curve: null,
      source_ref: `synthetic-contract-${month}`,
      expected_version: version++,
    })
  }
  void dataset
  return ledger
}

export async function runL2(
  dataset: Dataset,
  datasetSha256: string,
  client: SkillClient,
  cfg: HarnessConfig = DEFAULT_CONFIG,
  clock: () => string = () => new Date().toISOString(),
): Promise<EvalRun> {
  if (cfg.holdoutFrom < cfg.window) throw new Error('holdoutFrom must be ≥ window')
  const end = cfg.holdoutDays > 0 ? Math.min(dataset.days.length, cfg.holdoutFrom + cfg.holdoutDays) : dataset.days.length
  const holdout = dataset.days.slice(cfg.holdoutFrom, end)
  const ledger = seedLedger(dataset, cfg, holdout)
  const skillVersions = Object.fromEntries((await client.skills()).map((s) => [s.id, s.version]))

  const perDay: DayResult[] = []
  for (let idx = cfg.holdoutFrom; idx < end; idx++) {
    const day = dataset.days[idx]!
    const history = dataset.days.slice(idx - cfg.window, idx)

    const forecasts = {} as Record<ForecastKind, ForecastBundle>
    for (const kind of ['LOAD', 'PV', 'PRICE'] as const) {
      forecasts[kind] = await client.forecast(kind, {
        kind,
        market_date: day.date,
        unit: kind === 'PRICE' ? 'yuan_per_mwh' : 'mw',
        history: history.map((h) => toCurve(h[FIELD[kind]] as string[], h.date)),
        exogenous: {},
        features_snapshot_ref: REF,
      })
    }

    const bounds = ledger.dayAheadBounds(day.date)
    const bid: BidOptimizationResult = await client.optimizeBid({
      market_date: day.date,
      price_forecast: forecasts.PRICE,
      adjustable_capacity_mw: toCurve(day.adjustable_capacity_mw, day.date),
      position_bounds: bounds,
      risk: cfg.risk,
    })

    let ledgerAccepted = false
    if (bid.solver.status === 'OPTIMAL') {
      try {
        ledger.append({
          id: `bid-${day.date}`,
          timescale: 'DAY_AHEAD',
          period: day.date,
          kind: 'BID_SUBMITTED',
          energy_mwh: bid.daily_energy_mwh,
          curve: bid.quantities_mwh,
          source_ref: `eval-${day.date}`,
          expected_version: ledger.read().version,
        })
        ledgerAccepted = true
      } catch {
        ledgerAccepted = false
      }
    }

    const actualLoad = nums(day.load_mw)
    const actualPv = nums(day.pv_mw)
    const actualPrice = nums(day.price_yuan_per_mwh)
    const load = bandOf(forecasts.LOAD)
    const pv = bandOf(forecasts.PV)
    const price = bandOf(forecasts.PRICE)
    const capMwh = nums(day.adjustable_capacity_mw).map((c) => c * Number(cfg.risk.commitment_buffer_k) * 0.25)
    const eMax = Number(bounds.daily_energy_max_mwh)
    const daylight = actualPv.map((v, i) => [v, i] as const).filter(([v]) => v > 0).map(([, i]) => i)

    perDay.push({
      date: day.date,
      load: { mape: q(mape(actualLoad, load.p50), 6), coverage: q(coverage(actualLoad, load.p10, load.p90), 6) },
      pv: {
        nrmse: q(nrmse(actualPv, pv.p50, Number(dataset.meta.pv_capacity_mw)), 6),
        coverage: q(
          coverage(
            daylight.map((i) => actualPv[i]!),
            daylight.map((i) => pv.p10[i]!),
            daylight.map((i) => pv.p90[i]!),
          ),
          6,
        ),
      },
      price: {
        mape: q(mape(actualPrice, price.p50), 6),
        coverage: q(coverage(actualPrice, price.p10, price.p90), 6),
        direction: q(directionAccuracy(actualPrice, price.p50), 6),
      },
      bid: {
        status: bid.solver.status,
        ledger_accepted: ledgerAccepted,
        energy_mwh: bid.daily_energy_mwh,
        revenue_skill: q(
          realisedRevenue({ offers: nums(bid.prices_yuan_per_mwh.values), quantities: nums(bid.quantities_mwh.values) }, actualPrice),
          2,
        ),
        revenue_naive: q(realisedRevenue(naiveBid(capMwh, eMax), actualPrice), 2),
        revenue_hindsight: q(
          realisedRevenue(hindsightBid(actualPrice, capMwh, eMax, Number(cfg.risk.min_block_mwh)), actualPrice),
          2,
        ),
      },
    })
  }

  const fm = (pick: (d: DayResult) => { mape?: number; nrmse?: number; coverage: number; direction?: number }): ForecastMetrics => {
    const rows = perDay.map(pick)
    const has = (k: 'mape' | 'nrmse' | 'direction') => rows.every((r) => r[k] !== undefined)
    return {
      days: rows.length,
      mape: has('mape') ? q(mean(rows.map((r) => r.mape!)), 6) : null,
      nrmse: has('nrmse') ? q(mean(rows.map((r) => r.nrmse!)), 6) : null,
      coverage_p10_p90: q(mean(rows.map((r) => r.coverage)), 6),
      direction_accuracy: has('direction') ? q(mean(rows.map((r) => r.direction!)), 6) : null,
    }
  }
  const sum = (f: (d: DayResult) => number) => q(perDay.reduce((s, d) => s + f(d), 0), 2)
  const skill = sum((d) => d.bid.revenue_skill)
  const naive = sum((d) => d.bid.revenue_naive)
  const hindsight = sum((d) => d.bid.revenue_hindsight)

  const body = {
    layer: 'L2' as const,
    dataset: { name: dataset.meta.name, sha256: datasetSha256, synthetic: dataset.meta.synthetic },
    skill_versions: skillVersions,
    config: cfg,
    metrics: {
      load: fm((d) => d.load),
      pv: fm((d) => d.pv),
      price: fm((d) => d.price),
      bid: {
        days: perDay.length,
        optimal_days: perDay.filter((d) => d.bid.status === 'OPTIMAL').length,
        ledger_accepted_days: perDay.filter((d) => d.bid.ledger_accepted).length,
        revenue_skill_yuan: skill,
        revenue_naive_yuan: naive,
        revenue_hindsight_yuan: hindsight,
        capture_ratio: hindsight === 0 ? 0 : q(skill / hindsight, 6),
        uplift_vs_naive: naive === 0 ? 0 : q(skill / naive, 6),
      },
    },
    per_day: perDay,
  }
  const digest = createHash('sha256').update(stableJson(body)).digest('hex')
  const runAt = clock()
  return { id: `l2-${runAt.replace(/[:.]/g, '-')}-${digest.slice(0, 8)}`, run_at: runAt, ...body, digest }
}