Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { buildMenubarPayload } from './menubar-json.js'
import { getDaysInRange, ensureCacheHydrated, emptyCache, BACKFILL_DAYS, toDateString } from './daily-cache.js'
import { aggregateProjectsIntoDays, buildPeriodDataFromDays, dateKey } from './day-aggregator.js'
import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js'
import { aggregateModelEfficiency } from './model-efficiency.js'
import { renderDashboard } from './dashboard.js'
import { parseDateRangeFlags } from './cli-date.js'
import { runOptimize, scanAndDetect } from './optimize.js'
Expand Down Expand Up @@ -208,6 +209,7 @@ function buildJsonReport(projects: ProjectSummary[], period: string, periodKey:
}))

const modelMap: Record<string, { calls: number; cost: number; inputTokens: number; outputTokens: number; cacheReadTokens: number; cacheWriteTokens: number }> = {}
const modelEfficiency = aggregateModelEfficiency(projects)
for (const sess of sessions) {
for (const [model, d] of Object.entries(sess.modelBreakdown)) {
if (!modelMap[model]) { modelMap[model] = { calls: 0, cost: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 } }
Expand All @@ -221,7 +223,21 @@ function buildJsonReport(projects: ProjectSummary[], period: string, periodKey:
}
const models = Object.entries(modelMap)
.sort(([, a], [, b]) => b.cost - a.cost)
.map(([name, { cost, ...rest }]) => ({ name, ...rest, cost: convertCost(cost) }))
.map(([name, { cost, ...rest }]) => {
const efficiency = modelEfficiency.get(name)
return {
name,
...rest,
cost: convertCost(cost),
editTurns: efficiency?.editTurns ?? 0,
oneShotTurns: efficiency?.oneShotTurns ?? 0,
oneShotRate: efficiency?.oneShotRate ?? null,
retryRate: efficiency?.retryRate ?? null,
costPerEdit: efficiency?.costPerEditUSD !== null && efficiency?.costPerEditUSD !== undefined
? convertCost(efficiency.costPerEditUSD)
: null,
}
})

const catMap: Record<string, { turns: number; cost: number; editTurns: number; oneShotTurns: number }> = {}
for (const sess of sessions) {
Expand Down
10 changes: 9 additions & 1 deletion src/dashboard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import React, { useState, useCallback, useEffect, useRef } from 'react'
import { render, Box, Text, useInput, useApp, useWindowSize } from 'ink'
import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js'
import { formatCost, formatTokens } from './format.js'
import { aggregateModelEfficiency } from './model-efficiency.js'
import { parseAllSessions, filterProjectsByName } from './parser.js'
import { loadPricing } from './models.js'
import { getAllProviders } from './providers/index.js'
Expand Down Expand Up @@ -313,10 +314,12 @@ function ProjectBreakdown({ projects, pw, bw, budgets }: { projects: ProjectSumm
const MODEL_COL_COST = 8
const MODEL_COL_CACHE = 7
const MODEL_COL_CALLS = 7
const MODEL_COL_ONESHOT = 7
const MODEL_NAME_WIDTH = 14

function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw: number; bw: number }) {
const modelTotals: Record<string, { calls: number; costUSD: number; freshInput: number; cacheRead: number; cacheWrite: number }> = {}
const modelEfficiency = aggregateModelEfficiency(projects)
for (const project of projects) {
for (const session of project.sessions) {
for (const [model, data] of Object.entries(session.modelBreakdown)) {
Expand All @@ -334,18 +337,23 @@ function ModelBreakdown({ projects, pw, bw }: { projects: ProjectSummary[]; pw:

return (
<Panel title="By Model" color={PANEL_COLORS.model} width={pw}>
<Text dimColor wrap="truncate-end">{''.padEnd(bw + 1 + MODEL_NAME_WIDTH)}{'cost'.padStart(MODEL_COL_COST)}{'cache'.padStart(MODEL_COL_CACHE)}{'calls'.padStart(MODEL_COL_CALLS)}</Text>
<Text dimColor wrap="truncate-end">{''.padEnd(bw + 1 + MODEL_NAME_WIDTH)}{'cost'.padStart(MODEL_COL_COST)}{'cache'.padStart(MODEL_COL_CACHE)}{'calls'.padStart(MODEL_COL_CALLS)}{'1shot'.padStart(MODEL_COL_ONESHOT)}</Text>
{sorted.map(([model, data], i) => {
const totalInput = data.freshInput + data.cacheRead + data.cacheWrite
const cacheHit = totalInput > 0 ? (data.cacheRead / totalInput) * 100 : 0
const cacheLabel = totalInput > 0 ? `${cacheHit.toFixed(1)}%` : '-'
const efficiency = modelEfficiency.get(model)
const oneShotLabel = efficiency?.oneShotRate !== null && efficiency?.oneShotRate !== undefined
? `${efficiency.oneShotRate.toFixed(1)}%`
: '-'
return (
<Text key={`${model}-${i}`} wrap="truncate-end">
<HBar value={data.costUSD} max={maxCost} width={bw} />
<Text> {fit(model, MODEL_NAME_WIDTH)}</Text>
<Text color={GOLD}>{formatCost(data.costUSD).padStart(MODEL_COL_COST)}</Text>
<Text>{cacheLabel.padStart(MODEL_COL_CACHE)}</Text>
<Text>{String(data.calls).padStart(MODEL_COL_CALLS)}</Text>
<Text>{oneShotLabel.padStart(MODEL_COL_ONESHOT)}</Text>
</Text>
)
})}
Expand Down
33 changes: 22 additions & 11 deletions src/export.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { dirname, join, resolve } from 'path'
import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js'
import { getCurrency, convertCost } from './currency.js'
import { dateKey } from './day-aggregator.js'
import { aggregateModelEfficiency } from './model-efficiency.js'

function escCsv(s: string): string {
const sanitized = /^[\t\r=+\-@]/.test(s) ? `'${s}` : s
Expand Down Expand Up @@ -105,6 +106,7 @@ function buildActivityRows(projects: ProjectSummary[], period: string): Row[] {

function buildModelRows(projects: ProjectSummary[], period: string): Row[] {
const modelTotals: Record<string, { calls: number; cost: number; input: number; output: number; cacheRead: number; cacheWrite: number }> = {}
const modelEfficiency = aggregateModelEfficiency(projects)
for (const project of projects) {
for (const session of project.sessions) {
for (const [model, d] of Object.entries(session.modelBreakdown)) {
Expand All @@ -123,17 +125,26 @@ function buildModelRows(projects: ProjectSummary[], period: string): Row[] {
return Object.entries(modelTotals)
.filter(([name]) => name !== '<synthetic>')
.sort(([, a], [, b]) => b.cost - a.cost)
.map(([model, d]) => ({
Period: period,
Model: model,
[`Cost (${code})`]: round2(convertCost(d.cost)),
'Share (%)': pct(d.cost, totalCost),
'API Calls': d.calls,
'Input Tokens': d.input,
'Output Tokens': d.output,
'Cache Read Tokens': d.cacheRead,
'Cache Write Tokens': d.cacheWrite,
}))
.map(([model, d]) => {
const efficiency = modelEfficiency.get(model)
return {
Period: period,
Model: model,
[`Cost (${code})`]: round2(convertCost(d.cost)),
'Share (%)': pct(d.cost, totalCost),
'API Calls': d.calls,
'Edit Turns': efficiency?.editTurns ?? 0,
'One-shot Rate (%)': efficiency?.oneShotRate ?? '',
'Retry Rate': efficiency?.retryRate ?? '',
[`Cost/Edit (${code})`]: efficiency?.costPerEditUSD !== null && efficiency?.costPerEditUSD !== undefined
? round2(convertCost(efficiency.costPerEditUSD))
: '',
'Input Tokens': d.input,
'Output Tokens': d.output,
'Cache Read Tokens': d.cacheRead,
'Cache Write Tokens': d.cacheWrite,
}
})
}

function buildToolRows(projects: ProjectSummary[]): Row[] {
Expand Down
59 changes: 59 additions & 0 deletions src/model-efficiency.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import { getShortModelName } from './models.js'
import type { ProjectSummary } from './types.js'

export type ModelEfficiency = {
model: string
editTurns: number
oneShotTurns: number
retries: number
editCostUSD: number
oneShotRate: number | null
retryRate: number | null
costPerEditUSD: number | null
}

type MutableModelEfficiency = Omit<ModelEfficiency, 'oneShotRate' | 'retryRate' | 'costPerEditUSD'>

function rate(num: number, den: number): number | null {
if (den === 0) return null
return Math.round((num / den) * 1000) / 10
}

export function aggregateModelEfficiency(projects: ProjectSummary[]): Map<string, ModelEfficiency> {
const byModel = new Map<string, MutableModelEfficiency>()

function ensure(model: string): MutableModelEfficiency {
let stats = byModel.get(model)
if (!stats) {
stats = { model, editTurns: 0, oneShotTurns: 0, retries: 0, editCostUSD: 0 }
byModel.set(model, stats)
}
return stats
}

for (const project of projects) {
for (const session of project.sessions) {
for (const turn of session.turns) {
if (!turn.hasEdits || turn.assistantCalls.length === 0) continue

const primaryModel = getShortModelName(turn.assistantCalls[0]!.model)
if (primaryModel === '<synthetic>') continue

const stats = ensure(primaryModel)
stats.editTurns++
if (turn.retries === 0) stats.oneShotTurns++
stats.retries += turn.retries
stats.editCostUSD += turn.assistantCalls.reduce((sum, call) => {
return call.model === '<synthetic>' ? sum : sum + call.costUSD
}, 0)
}
}
}

return new Map([...byModel.entries()].map(([model, stats]) => [model, {
...stats,
oneShotRate: rate(stats.oneShotTurns, stats.editTurns),
retryRate: stats.editTurns > 0 ? Math.round((stats.retries / stats.editTurns) * 10) / 10 : null,
costPerEditUSD: stats.editTurns > 0 ? stats.editCostUSD / stats.editTurns : null,
}]))
}
18 changes: 18 additions & 0 deletions tests/export.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,24 @@ describe('exportCsv', () => {
expect(projects).toContain("'\rcmd")
})

it('includes per-model efficiency metrics', async () => {
const periods: PeriodExport[] = [
{
label: '30 Days',
projects: [makeProject('app')],
},
]

const outputPath = join(tmpDir, 'models.csv')
const folder = await exportCsv(periods, outputPath)
const models = await readFile(join(folder, 'models.csv'), 'utf-8')

expect(models).toContain('Edit Turns')
expect(models).toContain('One-shot Rate (%)')
expect(models).toContain('Cost/Edit')
expect(models).toContain(',1,100,0,')
})

it('does not crash when periods array is empty', async () => {
const outputPath = join(tmpDir, 'empty.csv')
const folder = await exportCsv([], outputPath)
Expand Down
100 changes: 100 additions & 0 deletions tests/model-efficiency.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import { describe, expect, it } from 'vitest'

import { aggregateModelEfficiency } from '../src/model-efficiency.js'
import type { ClassifiedTurn, ParsedApiCall, ProjectSummary, SessionSummary } from '../src/types.js'

function call(model: string, costUSD = 1): ParsedApiCall {
return {
provider: 'claude',
model,
usage: {
inputTokens: 100,
outputTokens: 50,
cacheCreationInputTokens: 0,
cacheReadInputTokens: 0,
cachedInputTokens: 0,
reasoningTokens: 0,
webSearchRequests: 0,
},
costUSD,
tools: ['Edit'],
mcpTools: [],
skills: [],
hasAgentSpawn: false,
hasPlanMode: false,
speed: 'standard',
timestamp: '2026-05-05T00:00:00Z',
bashCommands: [],
deduplicationKey: `${model}-${costUSD}`,
}
}

function turn(model: string, opts: { hasEdits?: boolean; retries?: number; costUSD?: number } = {}): ClassifiedTurn {
return {
userMessage: '',
assistantCalls: [call(model, opts.costUSD ?? 1)],
timestamp: '2026-05-05T00:00:00Z',
sessionId: 's1',
category: 'coding',
retries: opts.retries ?? 0,
hasEdits: opts.hasEdits ?? true,
}
}

function project(turns: ClassifiedTurn[]): ProjectSummary {
const session: SessionSummary = {
sessionId: 's1',
project: 'app',
firstTimestamp: '2026-05-05T00:00:00Z',
lastTimestamp: '2026-05-05T00:00:00Z',
totalCostUSD: turns.reduce((sum, t) => sum + t.assistantCalls.reduce((s, c) => s + c.costUSD, 0), 0),
totalInputTokens: 0,
totalOutputTokens: 0,
totalCacheReadTokens: 0,
totalCacheWriteTokens: 0,
apiCalls: turns.reduce((sum, t) => sum + t.assistantCalls.length, 0),
turns,
modelBreakdown: {},
toolBreakdown: {},
mcpBreakdown: {},
bashBreakdown: {},
categoryBreakdown: {} as SessionSummary['categoryBreakdown'],
skillBreakdown: {},
}
return {
project: 'app',
projectPath: '/app',
sessions: [session],
totalCostUSD: session.totalCostUSD,
totalApiCalls: session.apiCalls,
}
}

describe('aggregateModelEfficiency', () => {
it('computes one-shot, retry, and cost-per-edit metrics by display model', () => {
const stats = aggregateModelEfficiency([project([
turn('claude-sonnet-4-5', { hasEdits: true, retries: 0, costUSD: 2 }),
turn('claude-sonnet-4-5', { hasEdits: true, retries: 2, costUSD: 4 }),
turn('claude-opus-4-6', { hasEdits: true, retries: 0, costUSD: 10 }),
turn('claude-sonnet-4-5', { hasEdits: false, retries: 0, costUSD: 3 }),
])])

const sonnet = stats.get('Sonnet 4.5')
expect(sonnet?.editTurns).toBe(2)
expect(sonnet?.oneShotTurns).toBe(1)
expect(sonnet?.oneShotRate).toBe(50)
expect(sonnet?.retryRate).toBe(1)
expect(sonnet?.costPerEditUSD).toBe(3)

const opus = stats.get('Opus 4.6')
expect(opus?.oneShotRate).toBe(100)
})

it('returns no stats for non-edit turns', () => {
const stats = aggregateModelEfficiency([project([
turn('claude-sonnet-4-5', { hasEdits: false }),
])])

expect(stats.size).toBe(0)
})
})
Loading