diff --git a/src/cli.ts b/src/cli.ts index 368cbbc..4647edd 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,6 +10,7 @@ import { buildMenubarPayload } from './menubar-json.js' import { getDaysInRange, ensureCacheHydrated, emptyCache, BACKFILL_DAYS, toDateString } from './daily-cache.js' import { aggregateProjectsIntoDays, buildPeriodDataFromDays, dateKey } from './day-aggregator.js' import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js' +import { aggregateModelEfficiency } from './model-efficiency.js' import { renderDashboard } from './dashboard.js' import { parseDateRangeFlags } from './cli-date.js' import { runOptimize, scanAndDetect } from './optimize.js' @@ -208,6 +209,7 @@ function buildJsonReport(projects: ProjectSummary[], period: string, periodKey: })) const modelMap: Record = {} + const modelEfficiency = aggregateModelEfficiency(projects) for (const sess of sessions) { for (const [model, d] of Object.entries(sess.modelBreakdown)) { if (!modelMap[model]) { modelMap[model] = { calls: 0, cost: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 } } @@ -221,7 +223,21 @@ function buildJsonReport(projects: ProjectSummary[], period: string, periodKey: } const models = Object.entries(modelMap) .sort(([, a], [, b]) => b.cost - a.cost) - .map(([name, { cost, ...rest }]) => ({ name, ...rest, cost: convertCost(cost) })) + .map(([name, { cost, ...rest }]) => { + const efficiency = modelEfficiency.get(name) + return { + name, + ...rest, + cost: convertCost(cost), + editTurns: efficiency?.editTurns ?? 0, + oneShotTurns: efficiency?.oneShotTurns ?? 0, + oneShotRate: efficiency?.oneShotRate ?? null, + retryRate: efficiency?.retryRate ?? null, + costPerEdit: efficiency?.costPerEditUSD !== null && efficiency?.costPerEditUSD !== undefined + ? convertCost(efficiency.costPerEditUSD) + : null, + } + }) const catMap: Record = {} for (const sess of sessions) { diff --git a/src/dashboard.tsx b/src/dashboard.tsx index c047d69..44ad353 100644 --- a/src/dashboard.tsx +++ b/src/dashboard.tsx @@ -4,6 +4,7 @@ import React, { useState, useCallback, useEffect, useRef } from 'react' import { render, Box, Text, useInput, useApp, useWindowSize } from 'ink' import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js' import { formatCost, formatTokens } from './format.js' +import { aggregateModelEfficiency } from './model-efficiency.js' import { parseAllSessions, filterProjectsByName } from './parser.js' import { loadPricing } from './models.js' import { getAllProviders } from './providers/index.js' @@ -313,10 +314,12 @@ function ProjectBreakdown({ projects, pw, bw, budgets }: { projects: ProjectSumm const MODEL_COL_COST = 8 const MODEL_COL_CACHE = 7 const MODEL_COL_CALLS = 7 +const MODEL_COL_ONESHOT = 7 const MODEL_NAME_WIDTH = 14 function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: number; bw: number }) { const modelTotals: Record = {} + const modelEfficiency = aggregateModelEfficiency(projects) for (const project of projects) { for (const session of project.sessions) { for (const [model, data] of Object.entries(session.modelBreakdown)) { @@ -334,11 +337,15 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: return ( - {''.padEnd(bw + 1 + MODEL_NAME_WIDTH)}{'cost'.padStart(MODEL_COL_COST)}{'cache'.padStart(MODEL_COL_CACHE)}{'calls'.padStart(MODEL_COL_CALLS)} + {''.padEnd(bw + 1 + MODEL_NAME_WIDTH)}{'cost'.padStart(MODEL_COL_COST)}{'cache'.padStart(MODEL_COL_CACHE)}{'calls'.padStart(MODEL_COL_CALLS)}{'1shot'.padStart(MODEL_COL_ONESHOT)} {sorted.map(([model, data], i) => { const totalInput = data.freshInput + data.cacheRead + data.cacheWrite const cacheHit = totalInput > 0 ? (data.cacheRead / totalInput) * 100 : 0 const cacheLabel = totalInput > 0 ? `${cacheHit.toFixed(1)}%` : '-' + const efficiency = modelEfficiency.get(model) + const oneShotLabel = efficiency?.oneShotRate !== null && efficiency?.oneShotRate !== undefined + ? `${efficiency.oneShotRate.toFixed(1)}%` + : '-' return ( @@ -346,6 +353,7 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: {formatCost(data.costUSD).padStart(MODEL_COL_COST)} {cacheLabel.padStart(MODEL_COL_CACHE)} {String(data.calls).padStart(MODEL_COL_CALLS)} + {oneShotLabel.padStart(MODEL_COL_ONESHOT)} ) })} diff --git a/src/export.ts b/src/export.ts index 4e1afc1..b00d567 100644 --- a/src/export.ts +++ b/src/export.ts @@ -4,6 +4,7 @@ import { dirname, join, resolve } from 'path' import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js' import { getCurrency, convertCost } from './currency.js' import { dateKey } from './day-aggregator.js' +import { aggregateModelEfficiency } from './model-efficiency.js' function escCsv(s: string): string { const sanitized = /^[\t\r=+\-@]/.test(s) ? `'${s}` : s @@ -105,6 +106,7 @@ function buildActivityRows(projects: ProjectSummary[], period: string): Row[] { function buildModelRows(projects: ProjectSummary[], period: string): Row[] { const modelTotals: Record = {} + const modelEfficiency = aggregateModelEfficiency(projects) for (const project of projects) { for (const session of project.sessions) { for (const [model, d] of Object.entries(session.modelBreakdown)) { @@ -123,17 +125,26 @@ function buildModelRows(projects: ProjectSummary[], period: string): Row[] { return Object.entries(modelTotals) .filter(([name]) => name !== '') .sort(([, a], [, b]) => b.cost - a.cost) - .map(([model, d]) => ({ - Period: period, - Model: model, - [`Cost (${code})`]: round2(convertCost(d.cost)), - 'Share (%)': pct(d.cost, totalCost), - 'API Calls': d.calls, - 'Input Tokens': d.input, - 'Output Tokens': d.output, - 'Cache Read Tokens': d.cacheRead, - 'Cache Write Tokens': d.cacheWrite, - })) + .map(([model, d]) => { + const efficiency = modelEfficiency.get(model) + return { + Period: period, + Model: model, + [`Cost (${code})`]: round2(convertCost(d.cost)), + 'Share (%)': pct(d.cost, totalCost), + 'API Calls': d.calls, + 'Edit Turns': efficiency?.editTurns ?? 0, + 'One-shot Rate (%)': efficiency?.oneShotRate ?? '', + 'Retry Rate': efficiency?.retryRate ?? '', + [`Cost/Edit (${code})`]: efficiency?.costPerEditUSD !== null && efficiency?.costPerEditUSD !== undefined + ? round2(convertCost(efficiency.costPerEditUSD)) + : '', + 'Input Tokens': d.input, + 'Output Tokens': d.output, + 'Cache Read Tokens': d.cacheRead, + 'Cache Write Tokens': d.cacheWrite, + } + }) } function buildToolRows(projects: ProjectSummary[]): Row[] { diff --git a/src/model-efficiency.ts b/src/model-efficiency.ts new file mode 100644 index 0000000..465dc61 --- /dev/null +++ b/src/model-efficiency.ts @@ -0,0 +1,59 @@ +import { getShortModelName } from './models.js' +import type { ProjectSummary } from './types.js' + +export type ModelEfficiency = { + model: string + editTurns: number + oneShotTurns: number + retries: number + editCostUSD: number + oneShotRate: number | null + retryRate: number | null + costPerEditUSD: number | null +} + +type MutableModelEfficiency = Omit + +function rate(num: number, den: number): number | null { + if (den === 0) return null + return Math.round((num / den) * 1000) / 10 +} + +export function aggregateModelEfficiency(projects: ProjectSummary[]): Map { + const byModel = new Map() + + function ensure(model: string): MutableModelEfficiency { + let stats = byModel.get(model) + if (!stats) { + stats = { model, editTurns: 0, oneShotTurns: 0, retries: 0, editCostUSD: 0 } + byModel.set(model, stats) + } + return stats + } + + for (const project of projects) { + for (const session of project.sessions) { + for (const turn of session.turns) { + if (!turn.hasEdits || turn.assistantCalls.length === 0) continue + + const primaryModel = getShortModelName(turn.assistantCalls[0]!.model) + if (primaryModel === '') continue + + const stats = ensure(primaryModel) + stats.editTurns++ + if (turn.retries === 0) stats.oneShotTurns++ + stats.retries += turn.retries + stats.editCostUSD += turn.assistantCalls.reduce((sum, call) => { + return call.model === '' ? sum : sum + call.costUSD + }, 0) + } + } + } + + return new Map([...byModel.entries()].map(([model, stats]) => [model, { + ...stats, + oneShotRate: rate(stats.oneShotTurns, stats.editTurns), + retryRate: stats.editTurns > 0 ? Math.round((stats.retries / stats.editTurns) * 10) / 10 : null, + costPerEditUSD: stats.editTurns > 0 ? stats.editCostUSD / stats.editTurns : null, + }])) +} diff --git a/tests/export.test.ts b/tests/export.test.ts index 91c2d4c..9555723 100644 --- a/tests/export.test.ts +++ b/tests/export.test.ts @@ -152,6 +152,24 @@ describe('exportCsv', () => { expect(projects).toContain("'\rcmd") }) + it('includes per-model efficiency metrics', async () => { + const periods: PeriodExport[] = [ + { + label: '30 Days', + projects: [makeProject('app')], + }, + ] + + const outputPath = join(tmpDir, 'models.csv') + const folder = await exportCsv(periods, outputPath) + const models = await readFile(join(folder, 'models.csv'), 'utf-8') + + expect(models).toContain('Edit Turns') + expect(models).toContain('One-shot Rate (%)') + expect(models).toContain('Cost/Edit') + expect(models).toContain(',1,100,0,') + }) + it('does not crash when periods array is empty', async () => { const outputPath = join(tmpDir, 'empty.csv') const folder = await exportCsv([], outputPath) diff --git a/tests/model-efficiency.test.ts b/tests/model-efficiency.test.ts new file mode 100644 index 0000000..80d8e59 --- /dev/null +++ b/tests/model-efficiency.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest' + +import { aggregateModelEfficiency } from '../src/model-efficiency.js' +import type { ClassifiedTurn, ParsedApiCall, ProjectSummary, SessionSummary } from '../src/types.js' + +function call(model: string, costUSD = 1): ParsedApiCall { + return { + provider: 'claude', + model, + usage: { + inputTokens: 100, + outputTokens: 50, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + }, + costUSD, + tools: ['Edit'], + mcpTools: [], + skills: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard', + timestamp: '2026-05-05T00:00:00Z', + bashCommands: [], + deduplicationKey: `${model}-${costUSD}`, + } +} + +function turn(model: string, opts: { hasEdits?: boolean; retries?: number; costUSD?: number } = {}): ClassifiedTurn { + return { + userMessage: '', + assistantCalls: [call(model, opts.costUSD ?? 1)], + timestamp: '2026-05-05T00:00:00Z', + sessionId: 's1', + category: 'coding', + retries: opts.retries ?? 0, + hasEdits: opts.hasEdits ?? true, + } +} + +function project(turns: ClassifiedTurn[]): ProjectSummary { + const session: SessionSummary = { + sessionId: 's1', + project: 'app', + firstTimestamp: '2026-05-05T00:00:00Z', + lastTimestamp: '2026-05-05T00:00:00Z', + totalCostUSD: turns.reduce((sum, t) => sum + t.assistantCalls.reduce((s, c) => s + c.costUSD, 0), 0), + totalInputTokens: 0, + totalOutputTokens: 0, + totalCacheReadTokens: 0, + totalCacheWriteTokens: 0, + apiCalls: turns.reduce((sum, t) => sum + t.assistantCalls.length, 0), + turns, + modelBreakdown: {}, + toolBreakdown: {}, + mcpBreakdown: {}, + bashBreakdown: {}, + categoryBreakdown: {} as SessionSummary['categoryBreakdown'], + skillBreakdown: {}, + } + return { + project: 'app', + projectPath: '/app', + sessions: [session], + totalCostUSD: session.totalCostUSD, + totalApiCalls: session.apiCalls, + } +} + +describe('aggregateModelEfficiency', () => { + it('computes one-shot, retry, and cost-per-edit metrics by display model', () => { + const stats = aggregateModelEfficiency([project([ + turn('claude-sonnet-4-5', { hasEdits: true, retries: 0, costUSD: 2 }), + turn('claude-sonnet-4-5', { hasEdits: true, retries: 2, costUSD: 4 }), + turn('claude-opus-4-6', { hasEdits: true, retries: 0, costUSD: 10 }), + turn('claude-sonnet-4-5', { hasEdits: false, retries: 0, costUSD: 3 }), + ])]) + + const sonnet = stats.get('Sonnet 4.5') + expect(sonnet?.editTurns).toBe(2) + expect(sonnet?.oneShotTurns).toBe(1) + expect(sonnet?.oneShotRate).toBe(50) + expect(sonnet?.retryRate).toBe(1) + expect(sonnet?.costPerEditUSD).toBe(3) + + const opus = stats.get('Opus 4.6') + expect(opus?.oneShotRate).toBe(100) + }) + + it('returns no stats for non-edit turns', () => { + const stats = aggregateModelEfficiency([project([ + turn('claude-sonnet-4-5', { hasEdits: false }), + ])]) + + expect(stats.size).toBe(0) + }) +})