Skip to content

Commit 0939961

Browse files
khromovkhromov
committed
wip
1 parent 0845ead commit 0939961

File tree

3 files changed

+117
-175
lines changed

3 files changed

+117
-175
lines changed

‎index.ts‎

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,8 @@ function calculateTotalCost(
242242

243243
/**
244244
* Resolve pricing lookup for a model
245-
* Returns the pricing lookup result or null if not found or disabled
245+
* Returns the pricing lookup result or null if disabled
246+
* Throws an error (exits process) if pricing cannot be found and is not disabled
246247
*/
247248
function resolvePricingLookup(modelString: string): ModelPricingLookup | null {
248249
const costDisabled = process.env.MODEL_COST_DISABLED === "true";
@@ -255,30 +256,50 @@ function resolvePricingLookup(modelString: string): ModelPricingLookup | null {
255256

256257
// Check if pricing data file exists
257258
if (!isPricingAvailable()) {
258-
console.warn(
259-
`⚠️ Model pricing file not found. Run 'bun run update-model-pricing' to download it.`,
259+
console.error(
260+
`\n✗ Model pricing file not found. Run 'bun run update-model-pricing' to download it.`,
260261
);
261-
console.warn(` Cost calculation will be skipped.`);
262-
return null;
262+
console.error(
263+
` Or set MODEL_COST_DISABLED=true to skip cost calculation.\n`,
264+
);
265+
process.exit(1);
263266
}
264267

265268
// If explicit cost name is provided, use that
266269
if (explicitCostName) {
267270
const lookup = lookupModelPricingByKey(explicitCostName);
268271
if (!lookup) {
269-
console.warn(
270-
`⚠️ Could not find pricing for MODEL_COST_NAME="${explicitCostName}"`,
272+
console.error(
273+
`\n✗ Could not find pricing for MODEL_COST_NAME="${explicitCostName}" in model-pricing.json`,
271274
);
272-
return null;
275+
console.error(
276+
` Check that the key exists in data/model-pricing.json.\n`,
277+
);
278+
process.exit(1);
273279
}
274280
return lookup;
275281
}
276282

277283
// Try automatic lookup
278284
const lookup = lookupModelPricing(modelString);
279285
if (!lookup) {
280-
console.warn(`⚠️ Could not find pricing for model "${modelString}"`);
281-
return null;
286+
console.error(
287+
`\n✗ Could not find pricing for model "${modelString}" in model-pricing.json`,
288+
);
289+
console.error(`\n Options:`);
290+
console.error(
291+
` 1. Set MODEL_COST_NAME=<key> to explicitly specify the pricing key`,
292+
);
293+
console.error(
294+
` Example: MODEL_COST_NAME=vercel_ai_gateway/anthropic/claude-sonnet-4`,
295+
);
296+
console.error(
297+
` 2. Set MODEL_COST_DISABLED=true to skip cost calculation`,
298+
);
299+
console.error(
300+
`\n Browse data/model-pricing.json to find the correct key for your model.\n`,
301+
);
302+
process.exit(1);
282303
}
283304

284305
return lookup;
@@ -435,10 +456,15 @@ async function main() {
435456
const isHttpTransport = mcpServerUrl && isHttpUrl(mcpServerUrl);
436457
const mcpTransportType = isHttpTransport ? "HTTP" : "StdIO";
437458

459+
const costDisabled = process.env.MODEL_COST_DISABLED === "true";
460+
438461
console.log("╔════════════════════════════════════════════════════╗");
439462
console.log("║ SvelteBench 2.0 - Multi-Test ║");
440463
console.log("╚════════════════════════════════════════════════════╝");
441464
console.log(`Model(s): ${models.join(", ")}`);
465+
if (costDisabled) {
466+
console.log(`Pricing: Disabled (MODEL_COST_DISABLED=true)`);
467+
}
442468
console.log(`MCP Integration: ${mcpEnabled ? "Enabled" : "Disabled"}`);
443469
if (mcpEnabled) {
444470
console.log(`MCP Transport: ${mcpTransportType}`);
@@ -464,6 +490,14 @@ async function main() {
464490
process.exit(1);
465491
}
466492

493+
// Pre-validate pricing for all models before starting any benchmarks
494+
// This ensures we fail fast if any model's pricing is missing
495+
const pricingLookups = new Map<string, ModelPricingLookup | null>();
496+
for (const modelId of models) {
497+
const pricingLookup = resolvePricingLookup(modelId);
498+
pricingLookups.set(modelId, pricingLookup);
499+
}
500+
467501
// Set up outputs directory
468502
setupOutputsDirectory();
469503

@@ -497,8 +531,8 @@ async function main() {
497531
console.log(`🤖 Running benchmark for model: ${modelId}`);
498532
console.log("═".repeat(50));
499533

500-
// Resolve pricing for this model
501-
const pricingLookup = resolvePricingLookup(modelId);
534+
// Get pre-validated pricing for this model
535+
const pricingLookup = pricingLookups.get(modelId) ?? null;
502536
if (pricingLookup) {
503537
console.log(`💰 Pricing mapped: ${pricingLookup.matchedKey}`);
504538
}

‎lib/pricing.test.ts‎

Lines changed: 51 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -9,107 +9,74 @@ import {
99
} from "./pricing.ts";
1010

1111
describe("generateLookupCandidates", () => {
12-
describe("openrouter provider (slash format)", () => {
13-
it("should generate candidates for openrouter/anthropic/model", () => {
14-
const candidates = generateLookupCandidates(
15-
"openrouter/anthropic/claude-sonnet-4",
16-
);
17-
expect(candidates).toContain("openrouter/anthropic/claude-sonnet-4");
18-
expect(candidates).toContain("anthropic/claude-sonnet-4");
19-
expect(candidates).toContain("claude-sonnet-4");
20-
});
21-
22-
it("should generate candidates for openrouter/openai/model", () => {
23-
const candidates = generateLookupCandidates("openrouter/openai/gpt-4o");
24-
expect(candidates).toContain("openrouter/openai/gpt-4o");
25-
expect(candidates).toContain("openai/gpt-4o");
26-
expect(candidates).toContain("gpt-4o");
27-
});
28-
29-
it("should handle openrouter with simple model name", () => {
30-
const candidates = generateLookupCandidates("openrouter/some-model");
31-
expect(candidates).toContain("openrouter/some-model");
32-
expect(candidates).toContain("some-model");
33-
});
34-
});
35-
36-
describe("anthropic provider (slash format)", () => {
37-
it("should generate candidates for anthropic/model", () => {
38-
const candidates = generateLookupCandidates("anthropic/claude-haiku-4-5");
39-
expect(candidates).toContain("anthropic/claude-haiku-4-5");
40-
expect(candidates).toContain("claude-haiku-4-5");
41-
});
42-
});
43-
44-
describe("Vercel AI Gateway format (colon separator)", () => {
45-
it("should generate candidates for anthropic:model", () => {
46-
const candidates = generateLookupCandidates("anthropic:claude-sonnet-4");
47-
expect(candidates).toContain("anthropic:claude-sonnet-4");
48-
expect(candidates).toContain("anthropic/claude-sonnet-4");
49-
expect(candidates).toContain("claude-sonnet-4");
12+
describe("Vercel AI Gateway format (provider/model)", () => {
13+
it("should generate candidates for alibaba/qwen-3-14b", () => {
14+
const candidates = generateLookupCandidates("alibaba/qwen-3-14b");
15+
expect(candidates).toEqual([
16+
"vercel_ai_gateway/alibaba/qwen-3-14b",
17+
"alibaba/qwen-3-14b",
18+
"qwen-3-14b",
19+
]);
5020
});
5121

52-
it("should generate candidates for openai:model", () => {
53-
const candidates = generateLookupCandidates("openai:gpt-4o");
54-
expect(candidates).toContain("openai:gpt-4o");
55-
expect(candidates).toContain("openai/gpt-4o");
56-
expect(candidates).toContain("gpt-4o");
22+
it("should generate candidates for anthropic/claude-sonnet-4", () => {
23+
const candidates = generateLookupCandidates("anthropic/claude-sonnet-4");
24+
expect(candidates).toEqual([
25+
"vercel_ai_gateway/anthropic/claude-sonnet-4",
26+
"anthropic/claude-sonnet-4",
27+
"claude-sonnet-4",
28+
]);
5729
});
5830

59-
it("should generate candidates for google:model", () => {
60-
const candidates = generateLookupCandidates("google:gemini-2.0-flash");
61-
expect(candidates).toContain("google:gemini-2.0-flash");
62-
expect(candidates).toContain("google/gemini-2.0-flash");
63-
expect(candidates).toContain("gemini-2.0-flash");
64-
});
65-
66-
it("should generate candidates for x-ai:model", () => {
67-
const candidates = generateLookupCandidates("x-ai:grok-2");
68-
expect(candidates).toContain("x-ai:grok-2");
69-
expect(candidates).toContain("x-ai/grok-2");
70-
expect(candidates).toContain("grok-2");
71-
});
72-
});
73-
74-
describe("openai provider (slash format)", () => {
75-
it("should generate candidates for openai/model", () => {
31+
it("should generate candidates for openai/gpt-4o", () => {
7632
const candidates = generateLookupCandidates("openai/gpt-4o");
77-
expect(candidates).toContain("openai/gpt-4o");
78-
expect(candidates).toContain("gpt-4o");
33+
expect(candidates).toEqual([
34+
"vercel_ai_gateway/openai/gpt-4o",
35+
"openai/gpt-4o",
36+
"gpt-4o",
37+
]);
7938
});
8039

81-
it("should handle openai/gpt-4o-mini", () => {
82-
const candidates = generateLookupCandidates("openai/gpt-4o-mini");
83-
expect(candidates).toContain("openai/gpt-4o-mini");
84-
expect(candidates).toContain("gpt-4o-mini");
40+
it("should generate candidates for google/gemini-2.0-flash", () => {
41+
const candidates = generateLookupCandidates("google/gemini-2.0-flash");
42+
expect(candidates).toEqual([
43+
"vercel_ai_gateway/google/gemini-2.0-flash",
44+
"google/gemini-2.0-flash",
45+
"gemini-2.0-flash",
46+
]);
8547
});
86-
});
8748

88-
describe("lmstudio provider", () => {
89-
it("should generate candidates for lmstudio/model", () => {
90-
const candidates = generateLookupCandidates("lmstudio/llama-3-8b");
91-
expect(candidates).toContain("lmstudio/llama-3-8b");
92-
expect(candidates).toContain("llama-3-8b");
49+
it("should generate candidates for x-ai/grok-2", () => {
50+
const candidates = generateLookupCandidates("x-ai/grok-2");
51+
expect(candidates).toEqual([
52+
"vercel_ai_gateway/x-ai/grok-2",
53+
"x-ai/grok-2",
54+
"grok-2",
55+
]);
9356
});
9457
});
9558

96-
describe("unknown provider", () => {
97-
it("should return only the original string for unknown provider", () => {
98-
const candidates = generateLookupCandidates("unknown/some-model");
99-
expect(candidates).toEqual(["unknown/some-model"]);
100-
});
101-
102-
it("should handle unknown provider with colon format", () => {
103-
const candidates = generateLookupCandidates("unknown:some-model");
104-
expect(candidates).toContain("unknown:some-model");
105-
expect(candidates).toContain("unknown/some-model");
59+
describe("nested paths (openrouter style)", () => {
60+
it("should handle openrouter/anthropic/claude-sonnet-4", () => {
61+
const candidates = generateLookupCandidates(
62+
"openrouter/anthropic/claude-sonnet-4",
63+
);
64+
expect(candidates).toEqual([
65+
"vercel_ai_gateway/openrouter/anthropic/claude-sonnet-4",
66+
"openrouter/anthropic/claude-sonnet-4",
67+
"anthropic/claude-sonnet-4",
68+
"claude-sonnet-4",
69+
]);
10670
});
10771
});
10872

10973
describe("no provider prefix", () => {
110-
it("should return only the original string when no separator", () => {
74+
it("should return only the original string with gateway prefix when no separator", () => {
11175
const candidates = generateLookupCandidates("claude-sonnet-4");
112-
expect(candidates).toEqual(["claude-sonnet-4"]);
76+
expect(candidates).toEqual([
77+
"vercel_ai_gateway/claude-sonnet-4",
78+
"claude-sonnet-4",
79+
]);
11380
});
11481
});
11582
});

‎lib/pricing.ts‎

Lines changed: 20 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -46,29 +46,6 @@ export interface ModelPricingLookup {
4646
matchedKey: string;
4747
}
4848

49-
/**
50-
* Provider normalization configuration
51-
* - strip: Remove the provider prefix when generating candidates
52-
* - keepNested: For nested paths like "openrouter/anthropic/model", also try "anthropic/model"
53-
*/
54-
interface ProviderConfig {
55-
strip: boolean;
56-
keepNested: boolean;
57-
}
58-
59-
const PROVIDER_CONFIGS: Record<string, ProviderConfig> = {
60-
openrouter: { strip: true, keepNested: true },
61-
anthropic: { strip: true, keepNested: false },
62-
openai: { strip: true, keepNested: false },
63-
lmstudio: { strip: true, keepNested: false },
64-
google: { strip: true, keepNested: false },
65-
meta: { strip: true, keepNested: false },
66-
mistral: { strip: true, keepNested: false },
67-
cohere: { strip: true, keepNested: false },
68-
"x-ai": { strip: true, keepNested: false },
69-
deepseek: { strip: true, keepNested: false },
70-
};
71-
7249
// Cache the loaded pricing data
7350
let pricingData: Record<string, unknown> | null = null;
7451

@@ -98,74 +75,38 @@ function loadPricingData(): Record<string, unknown> {
9875
}
9976

10077
/**
101-
* Normalize a model string by converting different separator formats
102-
* Vercel AI Gateway uses "provider:model" format
103-
* Old multi-provider setup used "provider/model" format
104-
* LiteLLM pricing data uses various formats
105-
*/
106-
function normalizeModelString(modelString: string): string {
107-
// Convert colon separator to slash for unified processing
108-
// e.g., "anthropic:claude-sonnet-4" -> "anthropic/claude-sonnet-4"
109-
return modelString.replace(":", "/");
110-
}
111-
112-
/**
113-
* Generate lookup candidates for a model string using provider configuration
78+
* Generate lookup candidates for a model string
11479
* Returns candidates in priority order (most specific first)
11580
*
116-
* Supports both formats:
117-
* - Vercel AI Gateway: "anthropic:claude-sonnet-4"
118-
* - Multi-provider: "anthropic/claude-sonnet-4"
81+
* Vercel AI Gateway model IDs are in the format "provider/model-name"
82+
* LiteLLM pricing data stores them as "vercel_ai_gateway/provider/model-name"
83+
*
84+
* Examples:
85+
* - "alibaba/qwen-3-14b" -> tries "vercel_ai_gateway/alibaba/qwen-3-14b", "alibaba/qwen-3-14b", "qwen-3-14b"
86+
* - "anthropic/claude-sonnet-4" -> tries "vercel_ai_gateway/anthropic/claude-sonnet-4", "anthropic/claude-sonnet-4", "claude-sonnet-4"
11987
*/
12088
function generateLookupCandidates(modelString: string): string[] {
12189
const candidates: string[] = [];
12290

123-
// Normalize the model string (convert : to /)
124-
const normalizedString = normalizeModelString(modelString);
125-
126-
// Find matching provider config
127-
const slashIndex = normalizedString.indexOf("/");
128-
if (slashIndex === -1) {
129-
// No provider prefix, just use as-is
130-
return [modelString, normalizedString].filter((v, i, a) => a.indexOf(v) === i);
131-
}
132-
133-
const provider = normalizedString.slice(0, slashIndex);
134-
const config = PROVIDER_CONFIGS[provider];
135-
const remainder = normalizedString.slice(slashIndex + 1);
136-
137-
// Always try the original string first
91+
// Primary: Try with vercel_ai_gateway prefix (how LiteLLM stores gateway models)
92+
candidates.push(`vercel_ai_gateway/${modelString}`);
93+
94+
// Secondary: Try the model string as-is
13895
candidates.push(modelString);
13996

140-
// Also try the normalized version (with / instead of :)
141-
if (normalizedString !== modelString) {
142-
candidates.push(normalizedString);
143-
}
144-
145-
if (!config) {
146-
// Unknown provider, try original strings only
147-
return candidates.filter((v, i, a) => a.indexOf(v) === i);
148-
}
149-
150-
if (config.strip) {
151-
// Try without our provider prefix (just the model name)
152-
candidates.push(remainder);
153-
}
154-
155-
if (config.keepNested) {
156-
// For nested paths like "anthropic/claude-model", also try just "claude-model"
157-
const nestedSlashIndex = remainder.indexOf("/");
97+
// Tertiary: If there's a provider prefix, try just the model name
98+
const slashIndex = modelString.indexOf("/");
99+
if (slashIndex !== -1) {
100+
const modelName = modelString.slice(slashIndex + 1);
101+
candidates.push(modelName);
102+
103+
// Also try nested paths (e.g., "openrouter/anthropic/claude" -> "anthropic/claude", "claude")
104+
const nestedSlashIndex = modelName.indexOf("/");
158105
if (nestedSlashIndex !== -1) {
159-
candidates.push(remainder.slice(nestedSlashIndex + 1));
106+
candidates.push(modelName.slice(nestedSlashIndex + 1));
160107
}
161108
}
162109

163-
// Also try with common LiteLLM prefixes
164-
// LiteLLM often uses "provider/model" format
165-
if (!normalizedString.startsWith(provider + "/")) {
166-
candidates.push(`${provider}/${remainder}`);
167-
}
168-
169110
// Remove duplicates while preserving order
170111
return candidates.filter((v, i, a) => a.indexOf(v) === i);
171112
}

0 commit comments

Comments
 (0)