vpp-ai-platform/packages/runtime/test/m5.test.ts
Thomas Bayes 381b6d3521
Some checks failed
ci / typescript (push) Has been cancelled
ci / python (push) Has been cancelled
ci / evals (push) Has been cancelled
M5: shadow run, kill-switch hierarchy L0–L4, KPI dashboard, shadow replay CLI
Shadow run (ROADMAP M5, phase-1 acceptance form): SHADOW runtime mode records
bids without submitting them, simulates the market answer from the actual
day-ahead clearing price, dispatches to the simulation gateway, runs the D+1
review, and scores every day shadow-vs-human-vs-hindsight (ShadowDayRecord)
with a lineage-completeness audit. KpiReport regenerated after every day per
docs/12 §4 definitions (C1/C2 placeholders as named config).

Kill switches (docs/13 §8): BreakerService with L0 permit revocation, L1
envelope suspension, L2 loss breaker (mark-to-market, reduce-only bids),
L3 channel breaker (bids fall back to the file channel, dispatch BLOCKED),
L4 AI-off (templates run, no Proposal created); abnormal-day protocol on
EXTREME situations; per-level authority (B8 placeholder); auditable drill.

Runtime: shadow-close workflow, shadow schedule entries, live-data ingestion
through the quality gate, human-bid ingestion, breaker/KPI/shadow endpoints
and insight cards, replay CLI (npm run shadow). Ledger, time series and
streak counters are file-backed so a multi-week run survives restarts.

Domain: HumanBidRecord, ShadowDayRecord, KpiReport, BreakerRecord, SHADOW
receipt channel; contracts, fixtures and pydantic models regenerated.
Services: L2 metrics moved from evals so the shadow run and the harness
share one implementation.

Fixes: envelope/permit validity compared ISO timestamps as strings
('…00Z' vs '…00.000Z'); L2 baseline was stale since M4 (skill_versions only,
metrics unchanged) — rewritten from the live service.

Docs: docs/14 shadow-run runbook (timeline, breaker trigger/authority/
recovery, KPI definitions as implemented); README and CLAUDE.md status.

Tests: 21 consecutive shadow days with complete lineage, KPI report, WIDEN
recommendation produced but not acted on; restart durability; drill; L2/L3/L4
and abnormal-day paths; API.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017wrZgPL9LoKaD69BpEQU4v
2026-09-02 22:54:14 -04:00

275 lines
19 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { existsSync, readdirSync } from 'node:fs'
import type { AddressInfo } from 'node:net'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import type { Envelope, ShadowDayRecord } from '@vpp/domain'
import { BreakerAuthorityError } from '@vpp/services'
import { createApi } from '../src/api.js'
import { insightCards } from '../src/cards.js'
import type { RuntimeOptions } from '../src/runtime.js'
import { ShadowRunner, localToUtc, runBreakerDrill } from '../src/shadow.js'
import { TriggerService } from '../src/trigger.js'
import { envelope, harness } from './helpers.js'
const OPS = { id: 'user-ops-lead', role: 'ops-lead' }
const RISK = { id: 'user-risk-officer', role: 'risk-officer' }
const ADMIN = { id: 'user-platform-admin', role: 'platform-admin' }
const dispatchEnvelope = (bounds: Record<string, string>): Envelope =>
envelope(bounds, { id: 'env-dispatch-001', scope: { proposal_type: 'DISPATCH_PLAN', timescales: ['DAY_AHEAD'], resource_set: 'pool-hubei-01' } })
/** n consecutive market dates from `from` (all inside March 2026, the month the harness has a contract for). */
const dates = (from: string, n: number): string[] =>
Array.from({ length: n }, (_, i) => new Date(new Date(`${from}T00:00:00Z`).getTime() + i * 86_400_000).toISOString().slice(0, 10))
const flat = (date: string, v: string) => ({ interval_minutes: 15 as const, date, values: Array(96).fill(v) as string[] })
/** Shadow-mode harness: docs/07 world + both envelopes + trigger consumers + a runner that drives the injected clock. */
async function shadowHarness(opts: { dataDir?: string; seed?: boolean; config?: RuntimeOptions['config'] } = {}) {
const h = await harness({
llm: null,
...(opts.dataDir ? { dataDir: opts.dataDir } : {}),
...(opts.seed === false ? { seed: false } : {}),
config: { mode: 'SHADOW', shadow: { fulfillment: { kind: 'FIXED', ratio: '0.97' }, pvCapacityMw: '30' }, ...opts.config },
})
if (opts.seed !== false) {
h.rt.ctx.envelopes.register(envelope({ max_energy_mwh: '1500.0' }))
h.rt.ctx.envelopes.register(dispatchEnvelope({ max_total_mw: '30.0' }))
}
const t = new TriggerService(h.rt)
t.startEventConsumers()
const runner = new ShadowRunner(t, { setClock: h.setNow })
/** The human trader's actual bid for the day: a flat price-taker block, labelled synthetic. */
const humanBid = (date: string) => t.onHumanBid({ id: `hb-${date}`, market_date: date, prices_yuan_per_mwh: flat(date, '0.00'), quantities_mwh: flat(date, '3.000'), source: 'SYNTHETIC_NAIVE', received_at: localToUtc(date, '09:00', -1) })
return { h, t, runner, humanBid }
}
describe('ROADMAP M5 acceptance: shadow run on the docs/07 loop', () => {
it('21 consecutive shadow days with complete lineage, KPI report auto-generated, one envelope-widening recommendation produced but not acted on', async () => {
const { h, t, runner, humanBid } = await shadowHarness()
const days = dates('2026-03-02', 21)
const records: ShadowDayRecord[] = []
for (const d of days) {
humanBid(d)
records.push((await runner.replayDay(d)).record)
}
// --- every day closed the full loop with complete lineage
expect(records.map((r) => r.lineage_gaps).flat()).toEqual([])
expect(records.every((r) => r.lineage_complete)).toBe(true)
expect(records.map((r) => r.shadow.outcome)).toEqual(Array(21).fill('RELEASED'))
expect(records.every((r) => r.award !== null && r.dispatch?.outcome === 'RELEASED' && r.execution?.simulated === true && r.execution.fulfillment_ratio === '0.970' && r.execution.within_band)).toBe(true)
expect(records.every((r) => r.human?.source === 'SYNTHETIC_NAIVE' && r.review_finding_id !== null && r.forecast.load_mape !== null && r.forecast.pv_nrmse !== null && r.decision_latency_ms !== null)).toBe(true)
// --- external effects were simulated: bids recorded, never submitted; dispatch to the simulation gateway
expect(h.rt.ctx.gateway.receipts.list().map((r) => r.value.channel)).toEqual(Array(21).fill('SHADOW'))
expect(h.rt.ctx.simulationGateway.receipts.list()).toHaveLength(21)
expect(readdirSync(join(h.dataDir, 'exports'))).toEqual([])
expect(h.rt.ctx.ledger.read().entries.filter((e) => e.kind === 'AWARD')).toHaveLength(21)
// --- KPI report auto-generated after every day, per docs/12 §4 definitions
const kpi = h.rt.ctx.kpiReports.list().map((r) => r.value).sort((a, b) => (a.generated_at < b.generated_at ? -1 : 1)).at(-1)!
expect(h.rt.ctx.events.list({ event_type: 'KpiReportGenerated' })).toHaveLength(21)
expect(kpi.shadow).toMatchObject({ days: 21, complete_days: 21, consecutive_complete_days: 21, first_date: '2026-03-02', last_date: '2026-03-22', released_days: 21, pending_days: 0, breaker_trips: 0 })
expect(kpi.kpis.map((k) => k.id)).toEqual(['FORECAST_LOAD_MAPE', 'FORECAST_PV_NRMSE', 'POTENTIAL_ACCURACY', 'DECISION_LATENCY_P95_MS', 'DISPATCH_SUCCESS_RATE', 'REVENUE_UPLIFT_VS_HUMAN', 'CROSS_REGION_MATCH'])
const by = Object.fromEntries(kpi.kpis.map((k) => [k.id, k]))
expect(by['FORECAST_LOAD_MAPE']!.samples).toBe(21)
expect(by['POTENTIAL_ACCURACY']).toMatchObject({ value: '1.000000', status: 'MEET', samples: 21 })
expect(by['DISPATCH_SUCCESS_RATE']).toMatchObject({ value: '1.000000', status: 'MEET', samples: 21 })
expect(by['DECISION_LATENCY_P95_MS']).toMatchObject({ value: '0.000000', status: 'MEET', samples: 21 }) // frozen test clock; real clock in live mode
expect(by['REVENUE_UPLIFT_VS_HUMAN']!.samples).toBe(21)
expect(by['REVENUE_UPLIFT_VS_HUMAN']!.value).not.toBeNull()
expect(by['CROSS_REGION_MATCH']!.status).toBe('NOT_APPLICABLE')
expect(kpi.comparison.days_with_human_baseline).toBe(21)
expect(Number(kpi.comparison.hindsight_yuan)).toBeGreaterThanOrEqual(Number(kpi.comparison.shadow_yuan))
// --- widening recommendation from shadow data (20 compliant days), surfaced for humans, envelopes untouched
expect(kpi.shadow.widen_recommendations).toBeGreaterThanOrEqual(1)
const widen = h.rt.ctx.envelopeRequests.list().map((r) => r.value).filter((r) => r.action === 'WIDEN')
expect(widen.map((r) => r.envelope_id).sort()).toEqual(expect.arrayContaining(['env-bid-001', 'env-dispatch-001']))
expect(records[19]!.envelope_recommendations.some((e) => e.action === 'WIDEN')).toBe(true)
expect(h.rt.ctx.envelopes.get('env-bid-001')!.bounds).toEqual({ max_energy_mwh: '1500.0' })
expect(h.rt.ctx.envelopes.get('env-dispatch-001')!.bounds).toEqual({ max_total_mw: '30.0' })
expect(h.rt.ctx.events.list({ event_type: 'EnvelopeChanged' })).toHaveLength(0)
expect(h.rt.ctx.caseDesk.inbox().filter((p) => p.run_id !== '' && p.workflow === 'envelopeReview').length).toBeGreaterThanOrEqual(2)
// --- projections
expect(insightCards(h.rt.ctx, 'OPERATIONS_DASHBOARD').map((c) => c.title)).toEqual(['D+1 review', 'KPI dashboard', 'Kill switches', 'Open cases'])
expect(insightCards(h.rt.ctx, 'TRADING_DESK').find((c) => c.title === 'Shadow vs human vs hindsight')!.metrics.map((m) => m.name)).toContain('human_realised_revenue')
await h.rt.close()
}, 120_000)
it('the shadow run survives a process restart: ledger, streaks and shadow records are file-backed', async () => {
const a = await shadowHarness({ config: { widenAfterCompliantDays: 4 } })
for (const d of dates('2026-03-02', 3)) {
a.humanBid(d)
await a.runner.replayDay(d)
}
expect(a.h.rt.ctx.envelopeRequests.list()).toHaveLength(0)
await a.h.rt.close()
const b = await shadowHarness({ dataDir: a.h.dataDir, seed: false, config: { widenAfterCompliantDays: 4 } })
expect(b.h.rt.ctx.ledger.read().entries.map((e) => e.kind)).toEqual(['CONTRACT', ...Array(3).fill(['BID_SUBMITTED', 'AWARD']).flat()])
expect(b.h.rt.ctx.shadowDays.list()).toHaveLength(3)
for (const d of dates('2026-03-05', 2)) {
b.humanBid(d)
await b.runner.replayDay(d)
}
const kpi = b.h.rt.ctx.kpiReports.list().map((r) => r.value).sort((x, y) => (x.generated_at < y.generated_at ? -1 : 1)).at(-1)!
expect(kpi.shadow.consecutive_complete_days).toBe(5)
// The compliant-day streak (3 before, 4th after the restart) crossed the widening threshold only because it was persisted.
expect(b.h.rt.ctx.envelopeRequests.list().map((r) => r.value.action)).toContain('WIDEN')
await b.h.rt.close()
}, 60_000)
it('live data enters through the quality gate; quarantined curves never reach the forecast history', async () => {
const { h, t } = await shadowHarness()
const out = t.onMarketData({ market_date: '2026-05-01', load_mw: Array(96).fill('0'), price_yuan_per_mwh: Array(96).fill('400.00'), source: 'test-feed' })
expect(out.accepted).toEqual(['price:da'])
expect(out.quarantined[0]).toMatchObject({ series: 'load:aggregate' })
expect(h.rt.ctx.timeseries.latest('load:aggregate', '2026-05-01')).toBeUndefined()
expect(h.rt.ctx.ingestion.quarantined()).toHaveLength(1)
expect(h.rt.ctx.events.list({ event_type: 'MarketDataIngested' })).toHaveLength(1)
await h.rt.close()
})
})
describe('kill-switch hierarchy L0–L4 (docs/13 §8)', () => {
it('drill: every level trips on drill objects, is verified where the chain feels it, and is reset — all in the event log', async () => {
const { h } = await shadowHarness()
const drill = await runBreakerDrill(h.rt.ctx, OPS)
expect(drill.steps.map((s) => [s.level, s.verified])).toEqual([['L0', true], ['L1', true], ['L2', true], ['L3', true], ['L4', true]])
expect(drill.all_verified).toBe(true)
expect(h.rt.ctx.breakers.state().map((b) => [b.level, b.status, b.trip_count, b.drill])).toEqual([['L0', 'ARMED', 1, true], ['L1', 'ARMED', 1, true], ['L2', 'ARMED', 1, true], ['L3', 'ARMED', 1, true], ['L4', 'ARMED', 1, true]])
expect(h.rt.ctx.events.list({ event_type: 'BreakerDrillStep' })).toHaveLength(5)
expect(h.rt.ctx.events.list({ event_type: 'BreakerTripped' })).toHaveLength(5)
expect(h.rt.ctx.events.list({ event_type: 'BreakerReset' })).toHaveLength(5)
expect(h.rt.ctx.events.list({ event_type: 'BreakerDrillCompleted' })[0]!.payload).toMatchObject({ all_verified: true })
// The drill left no live effect: the next bid still auto-approves and releases.
const run = await h.rt.mastra.getWorkflow('dayAheadBid').createRun()
const res = await run.start({ inputData: { market_date: '2026-03-15' } })
expect(res.status === 'success' && res.result.lifecycle?.outcome).toBe('RELEASED')
// Agents and unauthorized roles cannot pull a switch (I1/I2, OPEN-QUESTION B8 mapping).
await expect(runBreakerDrill(h.rt.ctx, { id: 'trading-agent', role: 'ops-lead' })).rejects.toThrow(BreakerAuthorityError)
expect(() => h.rt.ctx.breakers.trip('L4', { id: 'user-t', role: 'senior-trader' }, 'x', null, h.rt.ctx.clock())).toThrow(BreakerAuthorityError)
expect(() => h.rt.ctx.breakers.reset('L2', { id: 'runtime', role: 'system' }, 'x', h.rt.ctx.clock())).toThrow(BreakerAuthorityError)
await h.rt.close()
})
it('L2 loss breaker: a day marked to market beyond the loss budget freezes auto-approval and allows only position-reducing bids until a human resets it', async () => {
// Marginal cost far above price → every cleared MWh loses money; simulation budget lifted so the chain releases the bid.
const { h, t, runner, humanBid } = await shadowHarness({
config: { risk: { risk_aversion: '0.3', commitment_buffer_k: '0.9', min_block_mwh: '0.5', marginal_cost_yuan_per_mwh: '100000' }, worstCaseLossBudgetYuan: '10000000000000' },
})
humanBid('2026-03-02')
const day1 = (await runner.replayDay('2026-03-02')).record
expect(day1.shadow.outcome).toBe('RELEASED')
expect(day1.breakers_tripped).toEqual(['L2'])
expect(h.rt.ctx.breakers.get('L2')).toMatchObject({ status: 'TRIPPED', tripped_by: { id: 'runtime', role: 'system' } })
expect(h.rt.ctx.breakers.get('L2').reason).toMatch(/expected loss .* exceeds daily budget 100000/)
expect(h.rt.ctx.events.list({ event_type: 'BreakerTripped' }).map((e) => (e.payload as { level: string }).level)).toEqual(['L2'])
// Next day: an exposure-increasing bid is rejected at rule check (reduce-only), so nothing reaches the envelope gate.
humanBid('2026-03-03')
const day2 = (await runner.replayDay('2026-03-03')).record
expect(day2.shadow.outcome).toBe('REJECTED')
const rejected = h.rt.ctx.events.list({ event_type: 'Lifecycle.REJECTED' }).at(-1)!.payload as { violations: Array<{ rule_id: string }> }
expect(rejected.violations.map((v) => v.rule_id)).toContain('breaker-l2-reduce-only')
expect(day2.award).toBeNull()
// Reset needs an authorized human and a basis; then autonomy returns.
expect(() => h.rt.ctx.breakers.reset('L2', OPS, '', h.rt.ctx.clock())).toThrow(/basis/)
h.rt.ctx.breakers.reset('L2', RISK, `loss reviewed in ${day1.review_finding_id}; budget restored`, h.rt.ctx.clock())
expect(h.rt.ctx.breakers.autoApprovalFreeze(h.rt.ctx.caseDesk.proposals.list()[0]!.value)).toBeNull()
expect(h.rt.ctx.events.list({ event_type: 'BreakerReset' })[0]!.payload).toMatchObject({ level: 'L2', by: RISK })
void t
await h.rt.close()
})
it('abnormal-day protocol: an EXTREME situation turns envelopes off for that date — the bid waits for a human', async () => {
const { h, t, runner, humanBid } = await shadowHarness()
h.skills.spread = 1.2 // p90/p50 = 2.2 ≥ extreme ratio 2.0 (OPEN-QUESTION B7)
humanBid('2026-03-02')
const day = (await runner.replayDay('2026-03-02')).record
expect(h.rt.ctx.events.list({ event_type: 'AbnormalDayDeclared' })[0]!.payload).toMatchObject({ market_date: '2026-03-02' })
expect(day.abnormal_day).toBe(true)
expect(day.shadow.outcome).toBe('PENDING_HUMAN')
expect(day.award).toBeNull()
const pending = h.rt.ctx.caseDesk.inbox().find((p) => p.run_id !== '' && p.workflow !== 'envelopeReview')!
expect(pending.reasons[0]).toMatch(/abnormal-day protocol for 2026-03-02/)
// A human can still approve the (unchanged) proposal — the protocol removes autonomy, not the chain.
const res = await t.decide(pending.run_id, { decision: 'approve', approver: OPS, comment: 'reviewed extreme-day forecast' })
expect(res.status === 'success' && res.result.outcome).toBe('RELEASED')
h.rt.ctx.breakers.clearAbnormalDay('2026-03-02', OPS, h.rt.ctx.clock())
expect(h.rt.ctx.breakers.isAbnormalDay('2026-03-02')).toBe(false)
await h.rt.close()
})
it('L3 channel breaker: bids fall back to the manual file channel, dispatch plans are authorized but BLOCKED', async () => {
const { h, t, humanBid } = await shadowHarness()
h.rt.ctx.breakers.trip('L3', OPS, 'trading-platform link down', '*', h.rt.ctx.clock())
humanBid('2026-03-02')
h.setNow(localToUtc('2026-03-02', '08:00', -1))
const run = await h.rt.mastra.getWorkflow('dayAheadBid').createRun()
const res = await run.start({ inputData: { market_date: '2026-03-02' } })
expect(res.status === 'success' && res.result.lifecycle?.outcome).toBe('RELEASED')
const receipt = h.rt.ctx.fileExportGateway.receipts.list()[0]!.value
expect(receipt.channel).toBe('FILE_EXPORT')
expect(existsSync(receipt.artifact_ref)).toBe(true)
expect(h.rt.ctx.gateway.receipts.list()).toHaveLength(0)
h.setNow(localToUtc('2026-03-02', '16:00', -1))
const cleared = await t.shadowClearing('2026-03-02')
expect(cleared?.decomposition?.result?.lifecycle?.outcome).toBe('BLOCKED')
expect(h.rt.ctx.events.list({ event_type: 'ChannelBlocked' })).toHaveLength(1)
expect(h.rt.ctx.simulationGateway.receipts.list()).toHaveLength(0)
await h.rt.close()
})
it('L4: AI suggestions off — the periodic templates still run the tools and produce data, but no Proposal is created', async () => {
const { h } = await shadowHarness()
h.rt.ctx.breakers.trip('L4', ADMIN, 'quarterly no-AI operation day', '*', h.rt.ctx.clock())
const run = await h.rt.mastra.getWorkflow('dayAheadBid').createRun()
const res = await run.start({ inputData: { market_date: '2026-03-15' } })
expect(res.status).toBe('success')
if (res.status !== 'success') return
expect(res.result.lifecycle_status).toBe('SUPPRESSED')
expect(res.result.proposal).toBeNull()
expect(h.skills.calls).toContain('milp') // the numbers were still computed and snapshotted
expect(h.rt.ctx.caseDesk.proposals.list()).toHaveLength(0)
expect(h.rt.ctx.events.list({ event_type: 'ProposalSuppressed' })).toHaveLength(1)
expect(h.rt.ctx.caseDesk.read(res.result.case_id).case.status).toBe('CLOSED_ABORTED')
h.rt.ctx.breakers.reset('L4', ADMIN, 'no-AI day complete', h.rt.ctx.clock())
const again = await (await h.rt.mastra.getWorkflow('dayAheadBid').createRun()).start({ inputData: { market_date: '2026-03-15' } })
expect(again.status === 'success' && again.result.lifecycle?.outcome).toBe('RELEASED')
await h.rt.close()
})
})
describe('Case Desk API: shadow, KPI and breaker endpoints', () => {
it('serves breakers/KPI/shadow days and enforces breaker authority', async () => {
const { h, t, runner, humanBid } = await shadowHarness()
const server = createApi(h.rt, t)
await new Promise<void>((r) => server.listen(0, '127.0.0.1', r))
const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const call = async (method: string, path: string, body?: unknown, headers: Record<string, string> = {}): Promise<{ status: number; body: any }> => {
const init: RequestInit = { method, headers: { 'content-type': 'application/json', ...headers } }
if (body !== undefined) init.body = JSON.stringify(body)
const res = await fetch(base + path, init)
return { status: res.status, body: await res.json() }
}
expect((await call('GET', '/health')).body).toMatchObject({ mode: 'SHADOW', breakers_tripped: [] })
expect((await call('GET', '/kpi')).status).toBe(404)
expect((await call('GET', '/breakers')).body.levels).toHaveLength(5)
expect((await call('POST', '/breakers/L2/trip', { reason: 'x' })).status).toBe(401)
expect((await call('POST', '/breakers/L2/trip', { reason: 'x' }, { 'x-user-id': 'user-t', 'x-user-role': 'senior-trader' })).status).toBe(403)
expect((await call('POST', '/breakers/L2/trip', { reason: 'manual risk hold' }, { 'x-user-id': RISK.id, 'x-user-role': RISK.role })).body).toMatchObject({ level: 'L2', status: 'TRIPPED' })
expect((await call('POST', '/breakers/L2/reset', { basis: 'hold lifted after review' }, { 'x-user-id': RISK.id, 'x-user-role': RISK.role })).body).toMatchObject({ level: 'L2', status: 'ARMED' })
expect((await call('POST', '/breakers/drill', undefined, { 'x-user-id': OPS.id, 'x-user-role': OPS.role })).body.all_verified).toBe(true)
expect((await call('POST', '/market-data', { market_date: '2026-05-02', price_yuan_per_mwh: Array(96).fill('410.00') })).body.accepted).toEqual(['price:da'])
humanBid('2026-03-02')
await runner.replayDay('2026-03-02')
expect((await call('GET', '/shadow/days')).body).toHaveLength(1)
expect((await call('GET', '/shadow/days/2026-03-02')).body.lineage_complete).toBe(true)
expect((await call('GET', '/kpi')).body.shadow.days).toBe(1)
expect((await call('GET', '/cards/OPERATIONS_DASHBOARD')).body.map((c: { title: string }) => c.title)).toContain('KPI dashboard')
await new Promise<void>((r) => server.close(() => r()))
await h.rt.close()
})
})