Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ Derived data pipelines, performance optimization, durable background jobs, enter
- [x] [P1] High-performer agent stack analysis: which combinations of MCPs, skills, commands, and subagents differentiate top performers
- [x] [P0] Per-tool success rate tracking with compound reliability computation (10 steps at 99% = 90.4% end-to-end)
- [x] [P0] Harness configuration fingerprinting from session telemetry (tools, context files, permissions, customizations)
- [ ] [P1] Context quality scoring: AGENTS.md freshness, token efficiency, guide/sensor coverage
- [x] [P1] Context quality scoring: AGENTS.md freshness, token efficiency, guide/sensor coverage
- [ ] [P1] Harness evolution timeline: before/after correlation of configuration changes with outcome changes
- [ ] [P1] Harnessability scoring per project: documentation quality, typing strength, module boundaries
- [ ] [P1] Paragon's 4-dimension evaluation: tool correctness, tool usage accuracy, task completion, task efficiency
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import { render, screen } from "@testing-library/react"

import { ContextQualityTable } from "@/components/maturity/context-quality-table"

describe("ContextQualityTable", () => {
it("renders an empty state", () => {
render(<ContextQualityTable rows={[]} />)

expect(screen.getByText("No context quality signals available yet.")).toBeInTheDocument()
})

it("renders context quality rows", () => {
render(
<ContextQualityTable
rows={[
{
repository: "acme/api",
session_count: 12,
context_quality_score: 82.8,
guide_coverage_score: 80,
guide_freshness_score: 100,
token_efficiency_score: 91.4,
sensor_coverage_score: 70,
cache_hit_rate: 0.42,
avg_input_tokens: 18_500,
context_usage_coverage_pct: 50,
tool_coverage_pct: 100,
model_coverage_pct: 75,
facet_coverage_pct: 55,
has_claude_md: true,
has_agents_md: false,
readiness_checked_at: "2026-04-20T00:00:00Z",
top_gaps: ["Add AGENTS.md", "Complete outcome facets"],
},
]}
/>,
)

expect(screen.getByText("acme/api")).toBeInTheDocument()
expect(screen.getByText("12 sessions")).toBeInTheDocument()
expect(screen.getByText("82.8")).toBeInTheDocument()
expect(screen.getByText("100 fresh")).toBeInTheDocument()
expect(screen.getByText("42% cache, 18.5K avg input")).toBeInTheDocument()
expect(screen.getByText("50% context / 75% models")).toBeInTheDocument()
expect(screen.getByText("Add AGENTS.md")).toBeInTheDocument()
expect(screen.getByText("Complete outcome facets")).toBeInTheDocument()
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ describe("MaturitySummary", () => {
agent_team_modes: [],
customization_outcomes: [],
project_readiness: [],
context_quality: [],
}}
/>,
)
Expand Down
117 changes: 117 additions & 0 deletions frontend/src/components/maturity/context-quality-table.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
import { Badge } from "@/components/ui/badge"
import { Card, CardContent, CardHeader } from "@/components/ui/card"
import { formatMetric, formatPercent, formatTokens } from "@/lib/utils"
import type { ContextQualityEntry } from "@/types/api"

interface ContextQualityTableProps {
rows: ContextQualityEntry[]
}

function formatCoverage(value: number): string {
return `${value.toFixed(0)}%`
}

export function ContextQualityTable({ rows }: ContextQualityTableProps) {
if (rows.length === 0) {
return (
<Card>
<CardHeader>
<h3 className="text-sm font-medium">Context Quality</h3>
</CardHeader>
<CardContent>
<p className="text-sm text-muted-foreground">
No context quality signals available yet.
</p>
</CardContent>
</Card>
)
}

return (
<Card>
<CardHeader>
<h3 className="text-sm font-medium">Context Quality</h3>
<p className="text-sm text-muted-foreground">
Scores project context by guidance coverage, freshness, token efficiency, and telemetry
sensor coverage.
</p>
</CardHeader>
<CardContent>
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead>
<tr className="border-b border-border text-left text-muted-foreground">
<th className="pb-2 font-medium">Repository</th>
<th className="pb-2 text-right font-medium">Score</th>
<th className="pb-2 text-right font-medium">Guides</th>
<th className="pb-2 text-right font-medium">Tokens</th>
<th className="pb-2 text-right font-medium">Sensors</th>
<th className="pb-2 font-medium">Top Gaps</th>
</tr>
</thead>
<tbody>
{rows.map((row) => (
<tr key={row.repository} className="border-b border-border/40">
<td className="py-2">
<div className="space-y-1">
<p className="font-mono text-xs">{row.repository}</p>
<p className="text-xs text-muted-foreground">{row.session_count} sessions</p>
</div>
</td>
<td className="py-2 text-right">
<div>
<p className="font-medium">{formatMetric(row.context_quality_score)}</p>
<p className="text-xs text-muted-foreground">overall</p>
</div>
</td>
<td className="py-2 text-right">
<div>
<p>{formatMetric(row.guide_coverage_score)}</p>
<p className="text-xs text-muted-foreground">
{formatMetric(row.guide_freshness_score, 0)} fresh
</p>
</div>
</td>
<td className="py-2 text-right">
<div>
<p>{formatMetric(row.token_efficiency_score)}</p>
<p className="text-xs text-muted-foreground">
{formatPercent(row.cache_hit_rate)} cache,{" "}
{row.avg_input_tokens == null
? "-"
: formatTokens(row.avg_input_tokens)}{" "}
avg input
</p>
</div>
</td>
<td className="py-2 text-right">
<div>
<p>{formatMetric(row.sensor_coverage_score)}</p>
<p className="text-xs text-muted-foreground">
{formatCoverage(row.context_usage_coverage_pct)} context /{" "}
{formatCoverage(row.model_coverage_pct)} models
</p>
</div>
</td>
<td className="py-2">
<div className="flex flex-wrap gap-2">
{row.top_gaps.length === 0 ? (
<span className="text-xs text-muted-foreground">No major gaps</span>
) : (
row.top_gaps.map((gap) => (
<Badge key={`${row.repository}:${gap}`} variant="outline">
{gap}
</Badge>
))
)}
</div>
</td>
</tr>
))}
</tbody>
</table>
</div>
</CardContent>
</Card>
)
}
6 changes: 5 additions & 1 deletion frontend/src/pages/maturity.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { TeamCustomizationLandscapeTable } from "@/components/maturity/team-cust
import { LeverageScoreTable } from "@/components/maturity/leverage-score-table"
import { LeverageTrendChart } from "@/components/maturity/leverage-trend-chart"
import { EffectivenessScatter } from "@/components/maturity/effectiveness-scatter"
import { ContextQualityTable } from "@/components/maturity/context-quality-table"
import { ProjectReadinessTable } from "@/components/maturity/project-readiness-table"
import { ToolAdoptionSummary } from "@/components/tools/tool-adoption-summary"
import { ToolAdoptionChart } from "@/components/tools/tool-adoption-chart"
Expand Down Expand Up @@ -151,7 +152,10 @@ export function MaturityPage({ teamId, dateRange }: MaturityPageProps) {
)}

{data && activeTab === "projects" && (
<ProjectReadinessTable data={data.project_readiness} />
<div className="space-y-6">
<ContextQualityTable rows={data.context_quality} />
<ProjectReadinessTable data={data.project_readiness} />
</div>
)}
</div>
)
Expand Down
21 changes: 21 additions & 0 deletions frontend/src/types/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2035,6 +2035,26 @@ export interface ProjectReadinessEntry {
session_count: number
}

export interface ContextQualityEntry {
repository: string
session_count: number
context_quality_score: number
guide_coverage_score: number
guide_freshness_score: number
token_efficiency_score: number
sensor_coverage_score: number
cache_hit_rate: number | null
avg_input_tokens: number | null
context_usage_coverage_pct: number
tool_coverage_pct: number
model_coverage_pct: number
facet_coverage_pct: number
has_claude_md: boolean
has_agents_md: boolean
readiness_checked_at: string | null
top_gaps: string[]
}

export interface MaturityAnalyticsResponse {
tool_categories: ToolCategoryBreakdown
engineer_profiles: EngineerLeverageProfile[]
Expand All @@ -2050,6 +2070,7 @@ export interface MaturityAnalyticsResponse {
agent_team_modes: AgentTeamModeSummary[]
customization_outcomes: CustomizationOutcomeAttribution[]
project_readiness: ProjectReadinessEntry[]
context_quality: ContextQualityEntry[]
sessions_analyzed: number
avg_leverage_score: number
avg_effectiveness_score: number | null
Expand Down
21 changes: 21 additions & 0 deletions src/primer/common/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -2454,6 +2454,26 @@ class ProjectReadinessEntry(BaseModel):
session_count: int


class ContextQualityEntry(BaseModel):
repository: str
session_count: int
context_quality_score: float
guide_coverage_score: float
guide_freshness_score: float
token_efficiency_score: float
sensor_coverage_score: float
cache_hit_rate: float | None = None
avg_input_tokens: float | None = None
context_usage_coverage_pct: float
tool_coverage_pct: float
model_coverage_pct: float
facet_coverage_pct: float
has_claude_md: bool
has_agents_md: bool
readiness_checked_at: datetime | None = None
top_gaps: list[str] = Field(default_factory=list)


class MaturityAnalyticsResponse(BaseModel):
tool_categories: ToolCategoryBreakdown
engineer_profiles: list[EngineerLeverageProfile]
Expand All @@ -2469,6 +2489,7 @@ class MaturityAnalyticsResponse(BaseModel):
agent_team_modes: list[AgentTeamModeSummary] = []
customization_outcomes: list[CustomizationOutcomeAttribution] = []
project_readiness: list[ProjectReadinessEntry]
context_quality: list[ContextQualityEntry] = Field(default_factory=list)
sessions_analyzed: int
avg_leverage_score: float
avg_effectiveness_score: float | None = None
Expand Down
Loading