8796faca63
- packages/domain: ForecastRequest, BidOptimizationRequest/Result,
ReportRequest, SkillReport (+ golden and invalid fixtures, exported to
contracts/ and regenerated as pydantic models).
- skills-py/vpp_skills: FastAPI service with versioned registry; load/PV/
price forecasts (same-day-type EWM point forecast, conformal residual
quantiles — coverage test as acceptance gate); bid-optimization MILP on
HiGHS (binary block participation, hard ledger energy bounds, exact
Decimal fit of the rounded curve inside the bounds, revenue distribution
over quantile paths); report generator whose every figure is a
{tool_call_id, path} reference, with a verifier. 48 tests incl. hypothesis
property test that bids respect ledger constraints.
- packages/services: LedgerService.dayAheadBounds (the P7 cascade band
handed to the optimizer); Decimal resolved once for CJS/ESM interop.
- packages/evals: L2 metrics (MAPE, nRMSE, coverage, direction accuracy,
naive/hindsight revenue baselines), HTTP skill client, rolling-origin
harness that pushes each bid through the real ledger, CLI with
--check/--write-baseline; committed baseline on the SYNTHETIC dataset
(no historical Hubei data yet — baselines measure the harness, not KPI).
- CI: evals job boots the skill service and fails on baseline digest drift.
- docs/open-questions: A6 (flexibility marginal cost = offer floor); A4/B6
wired as placeholders. README/CLAUDE.md status → M2 done, M3 next.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UoYoGYzHkFyv3ALenkRPhA
62 lines
2.0 KiB
TypeScript
62 lines
2.0 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
|
import {
|
|
coverage,
|
|
directionAccuracy,
|
|
hindsightBid,
|
|
mape,
|
|
naiveBid,
|
|
nrmse,
|
|
realisedRevenue,
|
|
} from '../src/metrics.js'
|
|
|
|
describe('forecast metrics', () => {
|
|
it('mape ignores zero actuals and is 0 for a perfect forecast', () => {
|
|
expect(mape([10, 0, 20], [10, 5, 20])).toBe(0)
|
|
expect(mape([10, 20], [11, 18])).toBeCloseTo((0.1 + 0.1) / 2)
|
|
})
|
|
|
|
it('nrmse normalises by capacity', () => {
|
|
expect(nrmse([0, 10], [0, 10], 20)).toBe(0)
|
|
expect(nrmse([10, 10], [12, 8], 20)).toBeCloseTo(2 / 20)
|
|
})
|
|
|
|
it('coverage counts inclusive band hits', () => {
|
|
expect(coverage([1, 2, 3, 4], [1, 1, 4, 1], [1, 3, 5, 3])).toBe(0.5)
|
|
})
|
|
|
|
it('direction accuracy scores pairwise ordering, not level', () => {
|
|
expect(directionAccuracy([1, 2, 3], [10, 20, 30])).toBe(1)
|
|
expect(directionAccuracy([1, 2, 3], [30, 20, 10])).toBe(0)
|
|
expect(directionAccuracy([1, 2, 3], [1, 3, 2])).toBeCloseTo(2 / 3)
|
|
})
|
|
})
|
|
|
|
describe('bid backtest baselines', () => {
|
|
const price = [100, 300, 200, 50]
|
|
const cap = [2, 2, 2, 2]
|
|
|
|
it('realised revenue clears only where offer ≤ price', () => {
|
|
expect(realisedRevenue({ offers: [150, 150, 150, 150], quantities: [1, 1, 1, 1] }, price)).toBe(500)
|
|
})
|
|
|
|
it('naive bid is flat, price-taking and meets the energy bound', () => {
|
|
const bid = naiveBid(cap, 4)
|
|
expect(bid.quantities).toEqual([1, 1, 1, 1])
|
|
expect(bid.offers.every((o) => o === 0)).toBe(true)
|
|
expect(naiveBid(cap, 100).quantities).toEqual(cap) // capped by capacity
|
|
})
|
|
|
|
it('hindsight bid fills highest-priced intervals first and bounds revenue above', () => {
|
|
const bid = hindsightBid(price, cap, 3, 0.5)
|
|
expect(bid.quantities).toEqual([0, 2, 1, 0])
|
|
const best = realisedRevenue(bid, price)
|
|
expect(best).toBe(800)
|
|
expect(realisedRevenue(naiveBid(cap, 3), price)).toBeLessThan(best)
|
|
})
|
|
|
|
it('hindsight honours the block size', () => {
|
|
const bid = hindsightBid(price, cap, 2.2, 0.5)
|
|
expect(bid.quantities).toEqual([0, 2, 0, 0]) // leftover 0.2 < block → not placed
|
|
})
|
|
})
|