Skip to content

Commit b18f37f

Browse files
shenrunzhangshenrunzhang
committed
Changed commercial/academic -> closed/open source. Added Chayan to leaderboard
1 parent ae04aad commit b18f37f

File tree

7 files changed

+89
-69
lines changed

7 files changed

+89
-69
lines changed

‎src/components/DeferralCurve.tsx‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,17 @@ import React from 'react';
22
import './DeferralCurve.css';
33

44
interface DeferralCurveProps {
5-
academicPoints: {
5+
openSourcePoints: {
66
[key: string]: { accuracy: number; cost_per_1k: number };
77
};
8-
commercialPoints: {
8+
closedSourcePoints: {
99
[key: string]: { accuracy: number; cost_per_1k: number };
1010
};
1111
}
1212

13-
const DeferralCurve: React.FC<DeferralCurveProps> = ({ academicPoints, commercialPoints }) => {
13+
const DeferralCurve: React.FC<DeferralCurveProps> = ({ openSourcePoints, closedSourcePoints }) => {
1414
// Extract all accuracy and cost values
15-
const allPoints = [...Object.values(academicPoints), ...Object.values(commercialPoints)];
15+
const allPoints = [...Object.values(openSourcePoints), ...Object.values(closedSourcePoints)];
1616
const accuracyValues = allPoints.map(p => p.accuracy);
1717
const costValues = allPoints.map(p => p.cost_per_1k);
1818

@@ -202,23 +202,23 @@ const DeferralCurve: React.FC<DeferralCurveProps> = ({ academicPoints, commercia
202202
return null;
203203
})}
204204

205-
{/* Academic routers */}
206-
{Object.entries(academicPoints).map(([name, point], index) => {
205+
{/* Open-source routers */}
206+
{Object.entries(openSourcePoints).map(([name, point], index) => {
207207
const x = scaleX(point.cost_per_1k);
208208
const y = scaleY(point.accuracy);
209209
const color = routerColors[index % routerColors.length];
210210

211-
return renderShape(x, y, 'circle', color, `academic-${name}`);
211+
return renderShape(x, y, 'circle', color, `open-source-${name}`);
212212
})}
213213

214-
{/* Commercial routers */}
215-
{Object.entries(commercialPoints).map(([name, point], index) => {
214+
{/* Closed-source routers */}
215+
{Object.entries(closedSourcePoints).map(([name, point], index) => {
216216
const x = scaleX(point.cost_per_1k);
217217
const y = scaleY(point.accuracy);
218218
const color =
219-
routerColors[(index + Object.keys(academicPoints).length) % routerColors.length];
219+
routerColors[(index + Object.keys(openSourcePoints).length) % routerColors.length];
220220

221-
return renderShape(x, y, 'triangle', color, `commercial-${name}`);
221+
return renderShape(x, y, 'triangle', color, `closed-source-${name}`);
222222
})}
223223

224224
{/* Oracle accuracy line */}
@@ -342,9 +342,9 @@ const DeferralCurve: React.FC<DeferralCurveProps> = ({ academicPoints, commercia
342342
{/* Legend */}
343343
<div className="deferral-legend-side">
344344
<div className="legend-section">
345-
<h4>Academic</h4>
345+
<h4>Open-Source</h4>
346346
<div className="legend-items">
347-
{Object.entries(academicPoints).map(([name, point], index) => {
347+
{Object.entries(openSourcePoints).map(([name, point], index) => {
348348
const color = routerColors[index % routerColors.length];
349349
return (
350350
<div key={name} className="legend-item">
@@ -359,11 +359,11 @@ const DeferralCurve: React.FC<DeferralCurveProps> = ({ academicPoints, commercia
359359
</div>
360360

361361
<div className="legend-section">
362-
<h4>Commercial</h4>
362+
<h4>Closed-Source</h4>
363363
<div className="legend-items">
364-
{Object.entries(commercialPoints).map(([name, point], index) => {
364+
{Object.entries(closedSourcePoints).map(([name, point], index) => {
365365
const color =
366-
routerColors[(index + Object.keys(academicPoints).length) % routerColors.length];
366+
routerColors[(index + Object.keys(openSourcePoints).length) % routerColors.length];
367367
return (
368368
<div key={name} className="legend-item">
369369
<svg className="legend-shape" width="16" height="16">

‎src/data/mockData.ts‎

Lines changed: 40 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -183,14 +183,23 @@ const rawRouterData = [
183183
'Robustness Score': 100.0,
184184
'Latency Score': 0.1863,
185185
},
186+
{
187+
'Router Name': 'chayan',
188+
'Arena Score': 63.83,
189+
'Optimal Selection Score': 43.03,
190+
'Optimal Cost Score': 43.75,
191+
'Optimal Acc. Score': 88.74,
192+
'Robustness Score': null,
193+
'Latency Score': null,
194+
}
186195
];
187196

188197
// Router metadata mapping
189198
const routerMetadata: Record<
190199
string,
191200
{
192201
name: string;
193-
type: 'academic' | 'commercial';
202+
type: 'open-source' | 'closed-source';
194203
description: string;
195204
affiliation: string;
196205
modelPool: string[];
@@ -200,7 +209,7 @@ const routerMetadata: Record<
200209
> = {
201210
RouterDC: {
202211
name: 'RouterDC',
203-
type: 'academic',
212+
type: 'open-source',
204213
description: 'Dual contrastive learning-based router with cost optimization',
205214
affiliation: 'SUSTech',
206215
modelPool: ['GPT-3.5', 'Claude-2', 'Gemini-Pro', 'Llama-2-13B'],
@@ -209,7 +218,7 @@ const routerMetadata: Record<
209218
},
210219
azure: {
211220
name: 'Azure',
212-
type: 'commercial',
221+
type: 'closed-source',
213222
description: "Microsoft Azure's model routing service",
214223
affiliation: 'Microsoft',
215224
modelPool: ['GPT-4', 'GPT-3.5', 'GPT-4-Turbo'],
@@ -218,7 +227,7 @@ const routerMetadata: Record<
218227
},
219228
carrot: {
220229
name: 'CARROT',
221-
type: 'academic',
230+
type: 'open-source',
222231
description: 'Cost-aware routing with dual contrastive learning approach',
223232
affiliation: 'UMich',
224233
modelPool: ['GPT-4', 'Claude-3', 'Gemini-Pro', 'Llama-2-70B'],
@@ -227,7 +236,7 @@ const routerMetadata: Record<
227236
},
228237
gpt5: {
229238
name: 'GPT-5',
230-
type: 'commercial',
239+
type: 'closed-source',
231240
description: "OpenAI's internal routing system for GPT model family",
232241
affiliation: 'OpenAI',
233242
modelPool: ['GPT-5', 'GPT-4', 'GPT-3.5'],
@@ -236,7 +245,7 @@ const routerMetadata: Record<
236245
},
237246
graphrouter: {
238247
name: 'GraphRouter',
239-
type: 'academic',
248+
type: 'open-source',
240249
description: 'Graph neural network-based routing with semantic understanding',
241250
affiliation: 'UIUC',
242251
modelPool: ['GPT-4', 'Claude-3', 'Gemini-Pro', 'Llama-2-70B', 'Mixtral-8x7B'],
@@ -245,7 +254,7 @@ const routerMetadata: Record<
245254
},
246255
mirt_bert: {
247256
name: 'MIRT-BERT',
248-
type: 'academic',
257+
type: 'open-source',
249258
description: 'Multi-item response theory with BERT embeddings',
250259
affiliation: 'USTC',
251260
modelPool: ['GPT-4', 'Claude-3', 'Gemini-Pro', 'Llama-2-70B'],
@@ -254,7 +263,7 @@ const routerMetadata: Record<
254263
},
255264
nirt_bert: {
256265
name: 'NIRT-BERT',
257-
type: 'academic',
266+
type: 'open-source',
258267
description: 'Neural item response theory with BERT-based routing',
259268
affiliation: 'USTC',
260269
modelPool: ['GPT-4', 'Claude-3', 'Gemini-Pro', 'Llama-2-70B'],
@@ -263,57 +272,66 @@ const routerMetadata: Record<
263272
},
264273
notdiamond: {
265274
name: 'NotDiamond',
266-
type: 'commercial',
267-
description: 'Commercial routing service with access to 60+ models',
275+
type: 'closed-source',
276+
description: 'Closed-source routing service with access to 60+ models',
268277
affiliation: 'NotDiamond',
269278
modelPool: ['GPT-4', 'Claude-3', 'Gemini-Pro', 'Llama-2-70B', 'Mixtral-8x7B', 'Qwen-72B'],
270279
paperUrl: 'https://notdiamond.ai',
271280
githubUrl: undefined,
272281
},
273282
routellm: {
274283
name: 'RouteLLM',
275-
type: 'academic',
284+
type: 'open-source',
276285
description: 'Binary selection between strong and weak models',
277286
affiliation: 'Berkeley',
278287
modelPool: ['GPT-4', 'GPT-3.5'],
279288
paperUrl: 'https://arxiv.org/abs/2024.xxxxx',
280289
githubUrl: 'https://github.com/example/routellm',
281290
},
282291
routerbench_knn: {
283-
name: 'KNN',
284-
type: 'academic',
292+
name: 'RouterBench-KNN',
293+
type: 'open-source',
285294
description: 'K-Nearest Neighbors-based router benchmark',
286-
affiliation: 'Academic',
295+
affiliation: 'Martian',
287296
modelPool: ['GPT-4', 'GPT-3.5', 'Claude-3'],
288-
paperUrl: 'https://arxiv.org/abs/2024.xxxxx',
289-
githubUrl: 'https://github.com/example/routerbench-knn',
297+
paperUrl: 'https://arxiv.org/pdf/2403.12031',
298+
githubUrl: 'https://github.com/withmartian/routerbench',
290299
},
291300
routerbench_mlp: {
292-
name: 'MLP',
293-
type: 'academic',
301+
name: 'RouterBench-MLP',
302+
type: 'open-source',
294303
description: 'Multi-Layer Perceptron-based router benchmark',
295-
affiliation: 'Academic',
304+
affiliation: 'Martian',
296305
modelPool: ['GPT-4', 'GPT-3.5', 'Claude-3'],
297306
paperUrl: 'https://arxiv.org/abs/2024.xxxxx',
298307
githubUrl: 'https://github.com/example/routerbench-mlp',
299308
},
300309
vllm: {
301310
name: 'vLLM-SR',
302-
type: 'commercial',
311+
type: 'open-source',
303312
description: 'vLLM-based routing service',
304313
affiliation: 'vLLM',
305314
modelPool: ['GPT-4', 'GPT-3.5'],
306315
paperUrl: 'https://vllm.ai',
307-
githubUrl: undefined,
316+
githubUrl: 'https://github.com/vllm-project/semantic-router',
308317
},
318+
chayan: {
319+
name: 'Chayan',
320+
type: 'open-source',
321+
description: 'Chayan-based routing service',
322+
affiliation: 'Adaptive Classifier',
323+
modelPool: ['GPT-4', 'GPT-3.5'],
324+
paperUrl: 'https://arxiv.org/abs/2024.xxxxx',
325+
githubUrl: 'https://huggingface.co/adaptive-classifier/chayan',
326+
}
309327
};
310328

311329
// Convert raw data to Router format and calculate overall ranks
312330
const routersWithRanks = rawRouterData.map(router => {
313331
const id = router['Router Name'].toLowerCase().replace(/[_\s]/g, '-');
314332
const metadata = routerMetadata[router['Router Name']] || {
315333
name: router['Router Name'],
316-
type: 'academic' as const,
334+
type: 'open-source' as const,
317335
description: `Router: ${router['Router Name']}`,
318336
affiliation: 'Unknown',
319337
modelPool: [],

‎src/pages/HomePage.tsx‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ const HomePage: React.FC = () => {
217217
<div>
218218
<h3>Automated Framework</h3>
219219
<p className="tab-subtitle">
220-
Automated evaluation pipeline supporting both open-source and commercial
220+
Automated evaluation pipeline supporting both open-source and closed-source
221221
routers with real-time leaderboard updates.
222222
</p>
223223
</div>
@@ -238,7 +238,7 @@ const HomePage: React.FC = () => {
238238
are added
239239
</li>
240240
<li>
241-
<strong>Multi-Platform:</strong> Support for both academic and commercial
241+
<strong>Multi-Platform:</strong> Support for both open-source and closed-source
242242
routers
243243
</li>
244244
<li>
@@ -259,8 +259,8 @@ const HomePage: React.FC = () => {
259259
<div>
260260
<h3>Fair Comparison</h3>
261261
<p className="tab-subtitle">
262-
Unified evaluation protocol enabling fair comparison between academic and
263-
commercial routers under consistent conditions.
262+
Unified evaluation protocol enabling fair comparison between open-source and
263+
closed-source routers under consistent conditions.
264264
</p>
265265
</div>
266266
</div>
@@ -276,7 +276,7 @@ const HomePage: React.FC = () => {
276276
protocols
277277
</li>
278278
<li>
279-
<strong>Academic & Commercial:</strong> Fair comparison across different
279+
<strong>Open-Source & Closed-Source:</strong> Fair comparison across different
280280
router types
281281
</li>
282282
<li>

‎src/pages/LeaderboardPage.css‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@
215215
.leaderboard-row > div {
216216
padding: 8px 12px;
217217
overflow: hidden;
218-
text-overflow: ellipsis;
218+
/* text-overflow: ellipsis; */
219219
white-space: nowrap;
220220
text-align: center;
221221
}
@@ -321,16 +321,18 @@
321321
}
322322

323323
.affiliation-col {
324-
display: flex;
325-
align-items: center;
326-
text-align: left;
327-
justify-content: flex-start;
324+
overflow-x: auto !important;
325+
white-space: nowrap !important;
326+
scrollbar-width: thin;
328327
}
329328

329+
330330
.affiliation {
331-
font-size: 0.9rem;
332-
color: #6b7280;
333-
font-weight: 500;
331+
display: inline-block;
332+
min-width: max-content; /* content defines its own width */
333+
padding-bottom: 4px;
334+
335+
334336
}
335337

336338
.router-info {
@@ -376,12 +378,12 @@
376378
text-align: center;
377379
}
378380

379-
.type-badge.academic {
381+
.type-badge.open-source {
380382
background: #3b82f6;
381383
color: white;
382384
}
383385

384-
.type-badge.commercial {
386+
.type-badge.closed-source {
385387
background: #10b981;
386388
color: white;
387389
}
@@ -740,11 +742,11 @@
740742
align-items: center;
741743
}
742744

743-
/* Prevent content like "commercial" or long affiliations from stretching columns */
745+
/* Prevent content like "closed-source" or long affiliations from stretching columns */
744746
.leaderboard-header > div,
745747
.leaderboard-row > div {
746748
overflow: hidden;
747-
text-overflow: ellipsis;
749+
/* text-overflow: ellipsis; */
748750
white-space: nowrap;
749751
}
750752

@@ -766,9 +768,9 @@
766768

767769
.type-badge {
768770
display: inline-block;
769-
width: 110px; /* ✅ fixed width */
771+
width: 200px; /* ✅ fixed width */
770772
text-align: center;
771-
overflow: hidden;
773+
/* overflow: hidden; */
772774
white-space: nowrap;
773775
}
774776

@@ -790,7 +792,7 @@
790792
.leaderboard-row > div {
791793
padding: 0.75rem 0.75rem !important;
792794
overflow: hidden;
793-
text-overflow: ellipsis;
795+
/* text-overflow: ellipsis; */
794796
white-space: nowrap;
795797
}
796798

0 commit comments

Comments
 (0)