diff --git a/ampere-cli/src/jvmTest/kotlin/link/socket/ampere/renderer/EventRendererTest.kt b/ampere-cli/src/jvmTest/kotlin/link/socket/ampere/renderer/EventRendererTest.kt index db48735d..d858f257 100644 --- a/ampere-cli/src/jvmTest/kotlin/link/socket/ampere/renderer/EventRendererTest.kt +++ b/ampere-cli/src/jvmTest/kotlin/link/socket/ampere/renderer/EventRendererTest.kt @@ -6,11 +6,13 @@ import kotlinx.datetime.Instant import kotlinx.datetime.TimeZone import kotlinx.datetime.toLocalDateTime import link.socket.ampere.agents.domain.event.Event -import link.socket.ampere.agents.domain.event.EventSource import link.socket.ampere.agents.domain.event.CognitiveStateSnapshot +import link.socket.ampere.agents.domain.event.EventSource +import link.socket.ampere.agents.domain.event.ProviderCallCompletedEvent import link.socket.ampere.agents.domain.event.SparkAppliedEvent import link.socket.ampere.agents.domain.event.SparkRemovedEvent import link.socket.ampere.agents.domain.Urgency +import link.socket.ampere.api.model.TokenUsage import org.junit.jupiter.api.Test import java.io.ByteArrayOutputStream import java.io.PrintStream @@ -181,6 +183,32 @@ class EventRendererTest { availableToolCount = availableToolCount ) + private fun providerCallCompletedEvent( + eventId: String = "evt-llm-1", + timestamp: Instant = Clock.System.now(), + source: EventSource = EventSource.Agent("agent-test"), + urgency: Urgency = Urgency.LOW, + usage: TokenUsage = TokenUsage( + inputTokens = 1247, + outputTokens = 892, + estimatedCost = 0.0031, + ), + latencyMs: Long = 340, + success: Boolean = true, + ): ProviderCallCompletedEvent = ProviderCallCompletedEvent( + eventId = eventId, + timestamp = timestamp, + eventSource = source, + urgency = urgency, + workflowId = "wf-render", + agentId = "agent-test", + providerId = "openai", + modelId = "gpt-4.1", + usage = usage, + latencyMs = latencyMs, + success = success, + ) + @Test fun `render TaskCreated event shows task ID, description, and assignment`() { val output = captureTerminalOutput { _, renderer -> @@ -265,6 +293,19 @@ class EventRendererTest { assertContains(lowUrgencyOutput, "[LOW]") } + @Test + fun `render completed provider call shows usage cost and latency`() { + val output = captureTerminalOutput { _, renderer -> + renderer.render(providerCallCompletedEvent()) + } + + val rendered = stripAnsi(output) + assertContains(rendered, "1,247 in / 892 out") + assertContains(rendered, "~\$0.0031") + assertContains(rendered, "340ms") + assertContains(rendered, "ProviderCallCompleted") + } + @Test fun `render shows event source for agent events`() { val output = captureTerminalOutput { _, renderer -> diff --git a/ampere-core/build.gradle.kts b/ampere-core/build.gradle.kts index 5b2d6a28..680eaede 100644 --- a/ampere-core/build.gradle.kts +++ b/ampere-core/build.gradle.kts @@ -242,7 +242,7 @@ android { } } -tasks.register("kotlinConfiguration") { +val kotlinConfiguration = tasks.register("kotlinConfiguration") { val generatedSources = File(layout.buildDirectory.asFile.get(), "generated/kotlin/config") generatedSources.mkdirs() kotlin.sourceSets.commonMain.get().kotlin.srcDirs(generatedSources) @@ -265,9 +265,13 @@ tasks.register("kotlinConfiguration") { ) } -tasks.findByName("build")?.dependsOn( - tasks.findByName("kotlinConfiguration"), -) +tasks.matching { task -> + task.name.startsWith("compile") && task.name.contains("Kotlin") +}.configureEach { + dependsOn(kotlinConfiguration) +} + +tasks.findByName("build")?.dependsOn(kotlinConfiguration) ktlint { android.set(true) diff --git a/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/event/TelemetryEvent.kt b/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/event/TelemetryEvent.kt index 1c07ede6..2acc79f5 100644 --- a/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/event/TelemetryEvent.kt +++ b/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/event/TelemetryEvent.kt @@ -1,5 +1,6 @@ package link.socket.ampere.agents.domain.event +import kotlin.math.roundToLong import kotlinx.datetime.Instant import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable @@ -86,10 +87,21 @@ data class ProviderCallCompletedEvent( ): String = buildString { append("LLM call ${if (success) "completed" else "failed"}: $providerId/$modelId") cognitivePhase?.let { append(" [${it.name}]") } - append(" (${latencyMs}ms") + append(" (") + var wroteDetail = false if (usage.inputTokens != null || usage.outputTokens != null) { - append(", in=${usage.inputTokens ?: "?"}, out=${usage.outputTokens ?: "?"}") + append( + "${usage.inputTokens?.formatGrouped() ?: "?"} in / ${usage.outputTokens?.formatGrouped() ?: "?"} out", + ) + wroteDetail = true } + usage.estimatedCost?.let { estimatedCost -> + if (wroteDetail) append(", ") + append("~${estimatedCost.formatUsd()}") + wroteDetail = true + } + if (wroteDetail) append(", ") + append("${latencyMs}ms") append(")") errorType?.let { append(" [$it]") } workflowId?.let { append(" workflow=$it") } @@ -101,3 +113,37 @@ data class ProviderCallCompletedEvent( const val EVENT_TYPE: EventType = "ProviderCallCompleted" } } + +private fun Int.formatGrouped(): String = toString().formatGroupedDigits() + +private fun Long.formatGrouped(): String = toString().formatGroupedDigits() + +private fun String.formatGroupedDigits(): String { + if (length <= 3) return this + + val firstGroupSize = if (length % 3 == 0) 3 else length % 3 + return buildString { + append(this@formatGroupedDigits.substring(0, firstGroupSize)) + var index = firstGroupSize + while (index < this@formatGroupedDigits.length) { + append(",") + append(this@formatGroupedDigits.substring(index, index + 3)) + index += 3 + } + } +} + +private fun Double.formatUsd(): String { + val scaled = (this * 10_000).roundToLong() + val absoluteScaled = if (scaled < 0) -scaled else scaled + val dollars = absoluteScaled / 10_000 + val fraction = (absoluteScaled % 10_000).toString().padStart(4, '0') + + return buildString { + if (scaled < 0) append("-") + append("$") + append(dollars.formatGrouped()) + append(".") + append(fraction) + } +} diff --git a/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/reasoning/AgentLLMService.kt b/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/reasoning/AgentLLMService.kt index 41968cfc..876b1d16 100644 --- a/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/reasoning/AgentLLMService.kt +++ b/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/reasoning/AgentLLMService.kt @@ -205,16 +205,13 @@ class AgentLLMService( } } + val usage = TokenUsageExtractor.fromOpenAiUsage(completion.usage) + emitCompletedTelemetry( routingContext = routingContext, providerId = effectiveConfig.provider.id, modelId = model.name, - usage = completion.usage?.let { usage -> - TokenUsage( - inputTokens = usage.promptTokens, - outputTokens = usage.completionTokens, - ) - } ?: TokenUsage(), + usage = usage, success = true, startedAt = startedAt, ) diff --git a/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/reasoning/TokenUsageExtractor.kt b/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/reasoning/TokenUsageExtractor.kt new file mode 100644 index 00000000..7d1101c9 --- /dev/null +++ b/ampere-core/src/commonMain/kotlin/link/socket/ampere/agents/domain/reasoning/TokenUsageExtractor.kt @@ -0,0 +1,51 @@ +package link.socket.ampere.agents.domain.reasoning + +import ai.koog.prompt.message.ResponseMetaInfo +import com.aallam.openai.api.core.Usage +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.doubleOrNull +import kotlinx.serialization.json.jsonPrimitive +import link.socket.ampere.api.model.TokenUsage + +/** + * Maps provider-specific usage metadata into AMPERE's stable [TokenUsage] contract. + * + * Cost remains nullable because not every provider reports it, and AMPERE does not + * currently maintain local pricing tables for inference. + */ +internal object TokenUsageExtractor { + + fun fromOpenAiUsage(usage: Usage?): TokenUsage = usage?.let { + TokenUsage( + inputTokens = it.promptTokens, + outputTokens = it.completionTokens, + ) + } ?: TokenUsage() + + fun fromResponseMetaInfo(metaInfo: ResponseMetaInfo?): TokenUsage = metaInfo?.let { + TokenUsage( + inputTokens = it.inputTokensCount, + outputTokens = it.outputTokensCount, + estimatedCost = extractEstimatedCost(it.metadata), + ) + } ?: TokenUsage() + + private fun extractEstimatedCost(metadata: JsonObject?): Double? { + if (metadata == null) return null + + val candidateKeys = listOf( + "estimatedCost", + "estimated_cost", + "costUsd", + "cost_usd", + "cost", + ) + + for (key in candidateKeys) { + val value = metadata[key]?.jsonPrimitive?.doubleOrNull + if (value != null) return value + } + + return null + } +} diff --git a/ampere-core/src/commonMain/kotlin/link/socket/ampere/api/service/EventService.kt b/ampere-core/src/commonMain/kotlin/link/socket/ampere/api/service/EventService.kt index 65a047d0..2c74d8d3 100644 --- a/ampere-core/src/commonMain/kotlin/link/socket/ampere/api/service/EventService.kt +++ b/ampere-core/src/commonMain/kotlin/link/socket/ampere/api/service/EventService.kt @@ -2,6 +2,7 @@ package link.socket.ampere.api.service import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.filterIsInstance import kotlinx.datetime.Instant import link.socket.ampere.agents.domain.event.Event import link.socket.ampere.agents.domain.event.FileSystemEvent @@ -143,6 +144,16 @@ interface EventService { } } +@link.socket.ampere.api.AmpereStableApi +fun EventService.routingEvents( + filters: EventRelayFilters = EventRelayFilters(), +): Flow = observe(filters).filterIsInstance() + +@link.socket.ampere.api.AmpereStableApi +fun EventService.completionEvents( + filters: EventRelayFilters = EventRelayFilters(), +): Flow = observe(filters).filterIsInstance() + @link.socket.ampere.api.AmpereStableApi enum class EventStreamFilter { ALL, diff --git a/ampere-core/src/commonTest/kotlin/link/socket/ampere/agents/domain/reasoning/TokenUsageExtractorTest.kt b/ampere-core/src/commonTest/kotlin/link/socket/ampere/agents/domain/reasoning/TokenUsageExtractorTest.kt new file mode 100644 index 00000000..64de736f --- /dev/null +++ b/ampere-core/src/commonTest/kotlin/link/socket/ampere/agents/domain/reasoning/TokenUsageExtractorTest.kt @@ -0,0 +1,62 @@ +package link.socket.ampere.agents.domain.reasoning + +import ai.koog.prompt.message.ResponseMetaInfo +import com.aallam.openai.api.core.Usage +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlinx.datetime.Clock +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import link.socket.ampere.api.model.TokenUsage + +class TokenUsageExtractorTest { + + @Test + fun `maps OpenAI usage into AMPERE token usage`() { + val usage = Usage( + promptTokens = 1247, + completionTokens = 892, + totalTokens = 2139, + ) + + val extracted = TokenUsageExtractor.fromOpenAiUsage(usage) + + assertEquals( + TokenUsage( + inputTokens = 1247, + outputTokens = 892, + ), + extracted, + ) + } + + @Test + fun `maps response metadata into AMPERE token usage including cost`() { + val metaInfo = ResponseMetaInfo.create( + clock = Clock.System, + totalTokensCount = 2139, + inputTokensCount = 1247, + outputTokensCount = 892, + metadata = buildJsonObject { + put("estimatedCost", 0.0031) + }, + ) + + val extracted = TokenUsageExtractor.fromResponseMetaInfo(metaInfo) + + assertEquals( + TokenUsage( + inputTokens = 1247, + outputTokens = 892, + estimatedCost = 0.0031, + ), + extracted, + ) + } + + @Test + fun `returns empty token usage when provider metadata is absent`() { + assertEquals(TokenUsage(), TokenUsageExtractor.fromOpenAiUsage(null)) + assertEquals(TokenUsage(), TokenUsageExtractor.fromResponseMetaInfo(null)) + } +} diff --git a/ampere-core/src/jvmTest/kotlin/link/socket/ampere/api/EventServiceExtensionsTest.kt b/ampere-core/src/jvmTest/kotlin/link/socket/ampere/api/EventServiceExtensionsTest.kt new file mode 100644 index 00000000..8d20e500 --- /dev/null +++ b/ampere-core/src/jvmTest/kotlin/link/socket/ampere/api/EventServiceExtensionsTest.kt @@ -0,0 +1,163 @@ +package link.socket.ampere.api + +import app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.take +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.UnconfinedTestDispatcher +import kotlinx.coroutines.test.runTest +import kotlinx.datetime.Clock +import link.socket.ampere.agents.domain.cognition.sparks.CognitivePhase +import link.socket.ampere.agents.domain.event.EventSource +import link.socket.ampere.agents.domain.event.ProviderCallCompletedEvent +import link.socket.ampere.agents.domain.event.RoutingEvent +import link.socket.ampere.agents.domain.routing.CognitiveRelayImpl +import link.socket.ampere.agents.domain.routing.RelayConfig +import link.socket.ampere.agents.domain.routing.RoutingContext +import link.socket.ampere.agents.domain.routing.RoutingRule +import link.socket.ampere.agents.events.EventRepository +import link.socket.ampere.agents.events.api.AgentEventApi +import link.socket.ampere.agents.events.bus.EventSerialBus +import link.socket.ampere.agents.events.relay.EventRelayServiceImpl +import link.socket.ampere.api.internal.DefaultEventService +import link.socket.ampere.api.model.TokenUsage +import link.socket.ampere.api.service.completionEvents +import link.socket.ampere.api.service.routingEvents +import link.socket.ampere.data.DEFAULT_JSON +import link.socket.ampere.db.Database +import link.socket.ampere.domain.ai.configuration.AIConfiguration_Default +import link.socket.ampere.domain.ai.model.AIModel_Gemini +import link.socket.ampere.domain.ai.model.AIModel_OpenAI +import link.socket.ampere.domain.ai.provider.AIProvider_Google +import link.socket.ampere.domain.ai.provider.AIProvider_OpenAI + +@OptIn(ExperimentalCoroutinesApi::class) +class EventServiceExtensionsTest { + + private val scope = TestScope(UnconfinedTestDispatcher()) + + private lateinit var driver: JdbcSqliteDriver + private lateinit var eventRepository: EventRepository + private lateinit var eventBus: EventSerialBus + private lateinit var eventApi: AgentEventApi + private lateinit var eventService: DefaultEventService + + @BeforeTest + fun setUp() { + driver = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) + Database.Schema.create(driver) + val database = Database(driver) + + eventRepository = EventRepository(DEFAULT_JSON, scope, database) + eventBus = EventSerialBus(scope) + eventApi = AgentEventApi( + agentId = "sdk-consumer", + eventRepository = eventRepository, + eventSerialBus = eventBus, + ) + eventService = DefaultEventService( + eventRelayService = EventRelayServiceImpl(eventBus, eventRepository), + eventRepository = eventRepository, + ) + } + + @AfterTest + fun tearDown() { + driver.close() + } + + @Test + fun `routingEvents exposes relay events through public event service`() = runTest { + val routingEvents = mutableListOf() + val relay = CognitiveRelayImpl( + initialConfig = RelayConfig( + rules = listOf( + RoutingRule.ByPhase( + CognitivePhase.PLAN, + AIConfiguration_Default( + provider = AIProvider_Google, + model = AIModel_Gemini.Flash_2_5, + ), + ), + ), + ), + eventBus = eventBus, + ) + + val job = launch { + eventService.routingEvents() + .take(1) + .toList(routingEvents) + } + + delay(100) + + relay.resolveWithMetadata( + context = RoutingContext( + agentId = "planner-agent", + phase = CognitivePhase.PLAN, + ), + fallbackConfiguration = AIConfiguration_Default( + provider = AIProvider_OpenAI, + model = AIModel_OpenAI.GPT_4_1, + ), + ) + + job.join() + + val event = assertIs(routingEvents.single()) + assertEquals("planner-agent", event.agentId) + assertEquals(CognitivePhase.PLAN, event.phase) + assertEquals("Google", event.decision.providerName) + } + + @Test + fun `completionEvents exposes provider completion telemetry through public event service`() = runTest { + val completionEvents = mutableListOf() + + val job = launch { + eventService.completionEvents() + .take(1) + .toList(completionEvents) + } + + delay(100) + + eventApi.publish( + ProviderCallCompletedEvent( + eventId = "llm-complete-1", + timestamp = Clock.System.now(), + eventSource = EventSource.Agent("sdk-consumer"), + workflowId = "wf-telemetry", + agentId = "planner-agent", + cognitivePhase = CognitivePhase.PLAN, + providerId = "openai", + modelId = AIModel_OpenAI.GPT_4_1.name, + usage = TokenUsage( + inputTokens = 1247, + outputTokens = 892, + estimatedCost = 0.0031, + ), + latencyMs = 340, + success = true, + ), + ) + + job.join() + + val event = completionEvents.single() + assertEquals("wf-telemetry", event.workflowId) + assertEquals(1247, event.usage.inputTokens) + assertEquals(892, event.usage.outputTokens) + assertEquals(0.0031, event.usage.estimatedCost) + assertEquals(340, event.latencyMs) + } +}