diff --git a/examples/ai/src/commands.ts b/examples/ai/src/commands.ts index 3f53a4e92..402a7e0a0 100644 --- a/examples/ai/src/commands.ts +++ b/examples/ai/src/commands.ts @@ -2,7 +2,7 @@ import { ActivityLike, IMessageActivity, SentActivity } from '@microsoft/teams.a import { OpenAIChatModel } from '@microsoft/teams.openai'; -import { ILogger } from '../../../packages/common/dist/logging/logger'; +import { ILogger } from '@microsoft/teams.common'; import { handleFeedbackLoop } from './feedback'; import { handleDocumentationSearch } from './simple-rag'; diff --git a/examples/ai/src/feedback.ts b/examples/ai/src/feedback.ts index f2a1b40b2..07a49a362 100644 --- a/examples/ai/src/feedback.ts +++ b/examples/ai/src/feedback.ts @@ -5,18 +5,32 @@ import { MessageActivity, SentActivity, } from '@microsoft/teams.api'; +import { IFeedbackProvider, SentMessageData, FeedbackScore } from '@microsoft/teams.apps'; +import { ConsoleLogger } from '@microsoft/teams.common'; -// This store would ideally be persisted in a database -export const storedFeedbackByMessageId = new Map< - string, - { - incomingMessage: string; - outgoingMessage: string; - likes: number; - dislikes: number; - feedbacks: string[]; +/** + * Simple in-memory feedback provider for demo purposes. + * In production, use a provider for LangSmith, LangFuse, Braintrust, or Azure AI Foundry. + */ +export class InMemoryFeedbackProvider implements IFeedbackProvider { + readonly messages = new Map(); + readonly feedback = new Map(); + private readonly log = new ConsoleLogger('@tests/ai/feedback-provider'); + + async logSentMessage(data: SentMessageData): Promise { + this.messages.set(data.messageId, data); + this.log.info(`Message logged for ${data.messageId}: input=${data.input}, output=${data.output}`); + } + + async logFeedback(messageId: string, score: FeedbackScore): Promise { + const existing = this.feedback.get(messageId) ?? []; + existing.push(score); + this.feedback.set(messageId, existing); + this.log.info(`Feedback logged for ${messageId}: ${score.reaction}`); } ->(); +} + +export const feedbackProvider = new InMemoryFeedbackProvider(); export const handleFeedbackLoop = async ( model: IChatModel, @@ -31,7 +45,7 @@ export const handleFeedbackLoop = async ( const result = await prompt.send(activity.text); if (result) { - const { id: sentMessageId } = await send( + await send( result.content != null ? new MessageActivity(result.content) .addAiGenerated() @@ -39,14 +53,5 @@ export const handleFeedbackLoop = async ( .addFeedback() : 'I did not generate a response.' ); - - storedFeedbackByMessageId.set(sentMessageId, { - incomingMessage: activity.text, - outgoingMessage: result.content ?? '', - likes: 0, - dislikes: 0, - feedbacks: [], - }); - } }; diff --git a/examples/ai/src/index.ts b/examples/ai/src/index.ts index c040cbbd3..f0f9bc55e 100644 --- a/examples/ai/src/index.ts +++ b/examples/ai/src/index.ts @@ -1,6 +1,6 @@ import { ChatPrompt } from '@microsoft/teams.ai'; import { MessageActivity } from '@microsoft/teams.api'; -import { App } from '@microsoft/teams.apps'; +import { App, FeedbackPlugin } from '@microsoft/teams.apps'; import { ConsoleLogger } from '@microsoft/teams.common'; import { DevtoolsPlugin } from '@microsoft/teams.dev'; import { OpenAIChatModel } from '@microsoft/teams.openai'; @@ -13,7 +13,7 @@ import { structuredOutputCommand, weatherCommand, } from './commands'; -import { storedFeedbackByMessageId } from './feedback'; +import { feedbackProvider } from './feedback'; import { handleDocumentationSearch } from './simple-rag'; import { handleStatefulConversation } from './stateful-prompts'; @@ -21,7 +21,7 @@ const logger = new ConsoleLogger('@tests/ai'); const app = new App({ logger, - plugins: [new DevtoolsPlugin()], + plugins: [new DevtoolsPlugin(), new FeedbackPlugin({ provider: feedbackProvider })], }); const model = new OpenAIChatModel({ @@ -144,27 +144,4 @@ app.on('message', async ({ send, activity, log }) => { await handleStatefulConversation(model, activity, send, log); }); -app.on('message.submit.feedback', async ({ activity, log }) => { - const { reaction, feedback: feedbackJson } = activity.value.actionValue; - if (activity.replyToId == null) { - log.warn(`No replyToId found for messageId ${activity.id}`); - return; - } - const existingFeedback = storedFeedbackByMessageId.get(activity.replyToId); - /** - * feedbackJson looks like: - * {"feedbackText":"Nice!"} - */ - if (!existingFeedback) { - log.warn(`No feedback found for messageId ${activity.id}`); - } else { - storedFeedbackByMessageId.set(activity.id, { - ...existingFeedback, - likes: existingFeedback.likes + (reaction === 'like' ? 1 : 0), - dislikes: existingFeedback.dislikes + (reaction === 'dislike' ? 1 : 0), - feedbacks: [...existingFeedback.feedbacks, feedbackJson], - }); - } -}); - app.start(process.env.PORT || 3978).catch(console.error); diff --git a/examples/ai/src/simple-rag.ts b/examples/ai/src/simple-rag.ts index b4ce144e6..5d1d5778b 100644 --- a/examples/ai/src/simple-rag.ts +++ b/examples/ai/src/simple-rag.ts @@ -3,7 +3,7 @@ import Fuse from 'fuse.js'; import { ChatPrompt, IChatModel } from '@microsoft/teams.ai'; import { ActivityLike, IMessageActivity, MessageActivity } from '@microsoft/teams.api'; -import { ILogger } from '../../../packages/common/dist/logging/logger'; +import { ILogger } from '@microsoft/teams.common'; interface IDocumentationItem { id: string; diff --git a/examples/ai/src/stateful-prompts.ts b/examples/ai/src/stateful-prompts.ts index 8b63ded35..b25e09aa7 100644 --- a/examples/ai/src/stateful-prompts.ts +++ b/examples/ai/src/stateful-prompts.ts @@ -1,7 +1,7 @@ import { ChatPrompt, IChatModel, Message } from '@microsoft/teams.ai'; import { ActivityLike, IMessageActivity, MessageActivity } from '@microsoft/teams.api'; -import { ILogger } from '../../../packages/common/dist/logging/logger'; +import { ILogger } from '@microsoft/teams.common'; // Simple in-memory store for conversation histories // In your application, it may be a good idea to use a more diff --git a/examples/ai/src/tool-calling.ts b/examples/ai/src/tool-calling.ts index 352d10aec..335c45062 100644 --- a/examples/ai/src/tool-calling.ts +++ b/examples/ai/src/tool-calling.ts @@ -1,7 +1,7 @@ import { ChatPrompt, IChatModel } from '@microsoft/teams.ai'; import { ActivityLike, IMessageActivity, SentActivity } from '@microsoft/teams.api'; -import { ILogger } from '../../../packages/common/dist/logging/logger'; +import { ILogger } from '@microsoft/teams.common'; interface IPokemonSearch { pokemonName: string; diff --git a/packages/apps/src/plugins/feedback/index.ts b/packages/apps/src/plugins/feedback/index.ts new file mode 100644 index 000000000..a10f8fe08 --- /dev/null +++ b/packages/apps/src/plugins/feedback/index.ts @@ -0,0 +1,7 @@ +export { FeedbackPlugin } from './plugin'; +export type { + IFeedbackProvider, + FeedbackPluginOptions, + SentMessageData, + FeedbackScore, +} from './types'; diff --git a/packages/apps/src/plugins/feedback/plugin.spec.ts b/packages/apps/src/plugins/feedback/plugin.spec.ts new file mode 100644 index 000000000..bbcd46325 --- /dev/null +++ b/packages/apps/src/plugins/feedback/plugin.spec.ts @@ -0,0 +1,281 @@ +import { IPluginActivityEvent } from '../../types/plugin/plugin-activity-event'; +import { IPluginActivitySentEvent } from '../../types/plugin/plugin-activity-sent-event'; + +import { FeedbackPlugin } from './plugin'; +import { IFeedbackProvider } from './types'; + +function mockProvider(): jest.Mocked { + return { + logSentMessage: jest.fn().mockResolvedValue(undefined), + logFeedback: jest.fn().mockResolvedValue(undefined), + }; +} + +function messageEvent(conversationId: string, text: string): IPluginActivityEvent { + return { + activity: { + type: 'message', + text, + id: `msg-${Date.now()}`, + from: { id: 'user-1', name: 'User' }, + recipient: { id: 'bot-1', name: 'Bot' }, + conversation: { id: conversationId }, + channelId: 'msteams', + serviceUrl: 'https://test.botframework.com', + }, + token: { + appId: 'test-app', + serviceUrl: 'https://test.botframework.com', + from: 'bot' as const, + fromId: 'test-from', + toString: () => 'test-token', + isExpired: () => false, + }, + serviceUrl: 'https://test.botframework.com', + channelId: 'msteams', + bot: { id: 'bot-1', name: 'Bot' }, + conversation: { id: conversationId }, + user: { id: 'user-1', name: 'User' }, + } as unknown as IPluginActivityEvent; +} + +function feedbackInvokeEvent(replyToId: string | undefined, reaction: 'like' | 'dislike', feedback: string): IPluginActivityEvent { + return { + activity: { + type: 'invoke', + name: 'message/submitAction', + id: `invoke-${Date.now()}`, + replyToId, + value: { + actionName: 'feedback', + actionValue: { reaction, feedback }, + }, + from: { id: 'user-1', name: 'User' }, + recipient: { id: 'bot-1', name: 'Bot' }, + conversation: { id: 'conv-1' }, + channelId: 'msteams', + serviceUrl: 'https://test.botframework.com', + }, + token: { + appId: 'test-app', + serviceUrl: 'https://test.botframework.com', + from: 'bot' as const, + fromId: 'test-from', + toString: () => 'test-token', + isExpired: () => false, + }, + serviceUrl: 'https://test.botframework.com', + channelId: 'msteams', + bot: { id: 'bot-1', name: 'Bot' }, + conversation: { id: 'conv-1' }, + user: { id: 'user-1', name: 'User' }, + } as unknown as IPluginActivityEvent; +} + +function sentEvent(conversationId: string, messageId: string, text: string, feedbackLoopEnabled: boolean): IPluginActivitySentEvent { + return { + activity: { + id: messageId, + type: 'message', + text, + channelData: feedbackLoopEnabled ? { feedbackLoopEnabled: true } : undefined, + }, + serviceUrl: 'https://test.botframework.com', + channelId: 'msteams', + bot: { id: 'bot-1', name: 'Bot' }, + conversation: { id: conversationId }, + user: { id: 'user-1', name: 'User' }, + } as unknown as IPluginActivitySentEvent; +} + +describe('FeedbackPlugin', () => { + it('should auto-capture user input on incoming message activities', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + await plugin.onActivity(messageEvent('conv-1', 'Hello AI')); + await plugin.onActivitySent(sentEvent('conv-1', 'sent-1', 'Hi there!', true)); + + expect(provider.logSentMessage).toHaveBeenCalledWith({ + messageId: 'sent-1', + input: 'Hello AI', + output: 'Hi there!', + }); + }); + + it('should pass correct messageId, input, output to provider.logSentMessage', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + await plugin.onActivity(messageEvent('conv-42', 'What is the weather?')); + await plugin.onActivitySent(sentEvent('conv-42', 'msg-abc-123', 'It is sunny!', true)); + + expect(provider.logSentMessage).toHaveBeenCalledTimes(1); + const call = provider.logSentMessage.mock.calls[0][0]; + expect(call.messageId).toBe('msg-abc-123'); + expect(call.input).toBe('What is the weather?'); + expect(call.output).toBe('It is sunny!'); + }); + + it('should intercept feedback activities and call provider.logFeedback with replyToId', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + await plugin.onActivity(feedbackInvokeEvent('original-msg-1', 'like', '{"feedbackText":"Great!"}')); + + expect(provider.logFeedback).toHaveBeenCalledWith('original-msg-1', { + reaction: 'like', + comment: '{"feedbackText":"Great!"}', + }); + }); + + it('should handle dislike feedback', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + await plugin.onActivity(feedbackInvokeEvent('msg-2', 'dislike', '{"feedbackText":"Not helpful"}')); + + expect(provider.logFeedback).toHaveBeenCalledWith('msg-2', { + reaction: 'dislike', + comment: '{"feedbackText":"Not helpful"}', + }); + }); + + it('should ignore non-invoke / non-feedback activities', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + // A regular invoke that is NOT feedback + const nonFeedbackInvoke = { + activity: { + type: 'invoke', + name: 'composeExtension/query', + id: 'invoke-other', + value: { commandId: 'search' }, + from: { id: 'user-1', name: 'User' }, + recipient: { id: 'bot-1', name: 'Bot' }, + conversation: { id: 'conv-1' }, + channelId: 'msteams', + serviceUrl: 'https://test.botframework.com', + }, + token: { + appId: 'test-app', + serviceUrl: 'https://test.botframework.com', + from: 'bot' as const, + fromId: 'test-from', + toString: () => 'test-token', + isExpired: () => false, + }, + serviceUrl: 'https://test.botframework.com', + channelId: 'msteams', + bot: { id: 'bot-1', name: 'Bot' }, + conversation: { id: 'conv-1' }, + user: { id: 'user-1', name: 'User' }, + } as unknown as IPluginActivityEvent; + + await plugin.onActivity(nonFeedbackInvoke); + + expect(provider.logFeedback).not.toHaveBeenCalled(); + expect(provider.logSentMessage).not.toHaveBeenCalled(); + }); + + it('should ignore sent activities without feedbackLoopEnabled', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + await plugin.onActivity(messageEvent('conv-1', 'Hello')); + await plugin.onActivitySent(sentEvent('conv-1', 'sent-1', 'Hi there!', false)); + + expect(provider.logSentMessage).not.toHaveBeenCalled(); + }); + + it('should warn when replyToId is missing on feedback activity', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + const mockWarn = jest.fn(); + (plugin as any).log.warn = mockWarn; + + await plugin.onActivity(feedbackInvokeEvent(undefined, 'like', '{"feedbackText":"test"}')); + + expect(mockWarn).toHaveBeenCalled(); + expect(provider.logFeedback).not.toHaveBeenCalled(); + }); + + it('should propagate provider errors', async () => { + const provider = mockProvider(); + const error = new Error('Provider connection failed'); + provider.logSentMessage.mockRejectedValue(error); + + const plugin = new FeedbackPlugin({ provider }); + + await plugin.onActivity(messageEvent('conv-1', 'Hello')); + await expect( + plugin.onActivitySent(sentEvent('conv-1', 'sent-1', 'Response', true)) + ).rejects.toThrow('Provider connection failed'); + }); + + it('should propagate logFeedback errors', async () => { + const provider = mockProvider(); + const error = new Error('Feedback logging failed'); + provider.logFeedback.mockRejectedValue(error); + + const plugin = new FeedbackPlugin({ provider }); + + await expect( + plugin.onActivity(feedbackInvokeEvent('msg-1', 'like', '{"feedbackText":"test"}')) + ).rejects.toThrow('Feedback logging failed'); + }); + + it('should omit input for proactive messages with no prior user message', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + // Send without a prior message activity + await plugin.onActivitySent(sentEvent('conv-new', 'sent-1', 'Proactive message', true)); + + expect(provider.logSentMessage).toHaveBeenCalledWith({ + messageId: 'sent-1', + input: undefined, + output: 'Proactive message', + }); + }); + + it('should clean up stored input after logging trace', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + await plugin.onActivity(messageEvent('conv-1', 'First message')); + await plugin.onActivitySent(sentEvent('conv-1', 'sent-1', 'First response', true)); + + expect(provider.logSentMessage).toHaveBeenCalledTimes(1); + + // Second sent without a new input should have undefined input + await plugin.onActivitySent(sentEvent('conv-1', 'sent-2', 'Second response', true)); + + expect(provider.logSentMessage).toHaveBeenCalledTimes(2); + expect(provider.logSentMessage.mock.calls[1][0].input).toBeUndefined(); + }); + + it('should track inputs per conversation independently', async () => { + const provider = mockProvider(); + const plugin = new FeedbackPlugin({ provider }); + + await plugin.onActivity(messageEvent('conv-A', 'Question A')); + await plugin.onActivity(messageEvent('conv-B', 'Question B')); + + await plugin.onActivitySent(sentEvent('conv-A', 'sent-A', 'Answer A', true)); + await plugin.onActivitySent(sentEvent('conv-B', 'sent-B', 'Answer B', true)); + + expect(provider.logSentMessage).toHaveBeenCalledWith({ + messageId: 'sent-A', + input: 'Question A', + output: 'Answer A', + }); + expect(provider.logSentMessage).toHaveBeenCalledWith({ + messageId: 'sent-B', + input: 'Question B', + output: 'Answer B', + }); + }); +}); diff --git a/packages/apps/src/plugins/feedback/plugin.ts b/packages/apps/src/plugins/feedback/plugin.ts new file mode 100644 index 000000000..35c68b99d --- /dev/null +++ b/packages/apps/src/plugins/feedback/plugin.ts @@ -0,0 +1,65 @@ +import { ConsoleLogger } from '@microsoft/teams.common'; + +import pkg from '../../../package.json'; +import { Plugin } from '../../types'; +import { IPluginActivityEvent } from '../../types/plugin/plugin-activity-event'; +import { IPluginActivitySentEvent } from '../../types/plugin/plugin-activity-sent-event'; + +import { FeedbackPluginOptions, IFeedbackProvider } from './types'; + +@Plugin({ + name: 'feedback', + version: pkg.version, + description: 'Auto-captures AI traces and feedback, piping to eval services', +}) +export class FeedbackPlugin { + private readonly provider: IFeedbackProvider; + private readonly log = new ConsoleLogger('@microsoft/teams.apps/plugins/feedback'); + private readonly inputs = new Map(); + + constructor(options: FeedbackPluginOptions) { + this.provider = options.provider; + } + + async onActivity(event: IPluginActivityEvent) { + const { activity } = event; + + if (activity.type === 'message') { + this.inputs.set(activity.conversation.id, activity.text); + return; + } + + if ( + activity.type === 'invoke' && + activity.name === 'message/submitAction' && + activity.value?.actionName === 'feedback' + ) { + const replyToId = activity.replyToId; + + if (!replyToId) { + this.log.warn(`No replyToId found for feedback activity ${activity.id}`); + return; + } + + const { reaction, feedback: comment } = activity.value.actionValue; + await this.provider.logFeedback(replyToId, { reaction, comment }); + } + } + + async onActivitySent(event: IPluginActivitySentEvent) { + const { activity } = event; + + if (!activity.channelData?.feedbackLoopEnabled) { + return; + } + + const input = this.inputs.get(event.conversation.id); + this.inputs.delete(event.conversation.id); + + await this.provider.logSentMessage({ + messageId: activity.id, + input, + output: (activity as any).text ?? '', + }); + } +} diff --git a/packages/apps/src/plugins/feedback/types.ts b/packages/apps/src/plugins/feedback/types.ts new file mode 100644 index 000000000..ba7c9e6db --- /dev/null +++ b/packages/apps/src/plugins/feedback/types.ts @@ -0,0 +1,36 @@ +/** Data logged when the bot sends a feedback-enabled message */ +export interface SentMessageData { + /** The Teams message ID — used as the correlation key */ + messageId: string; + /** The user's input message, if available (absent for proactive messages) */ + input?: string; + /** The bot's AI-generated output */ + output: string; + /** Optional metadata (model name, token usage, etc.) */ + metadata?: Record; +} + +/** Feedback score sent to the eval service */ +export interface FeedbackScore { + reaction: 'like' | 'dislike'; + /** Raw feedback string from Teams, e.g. '{"feedbackText":"Nice!"}' */ + comment: string; +} + +/** + * Provider interface — implement this for each eval service. + * The messageId is used as the correlation key between trace and feedback. + * Providers like LangSmith/LangFuse support custom IDs natively. + * Foundry provider handles OpenTelemetry span context internally. + */ +export interface IFeedbackProvider { + /** Log a sent message (AI input/output). Use messageId as the correlation key. */ + logSentMessage(data: SentMessageData): Promise; + /** Attach feedback to a previously logged message by messageId. */ + logFeedback(messageId: string, score: FeedbackScore): Promise; +} + +/** Plugin constructor options */ +export interface FeedbackPluginOptions { + provider: IFeedbackProvider; +} diff --git a/packages/apps/src/plugins/index.ts b/packages/apps/src/plugins/index.ts index c202386ae..980c9467e 100644 --- a/packages/apps/src/plugins/index.ts +++ b/packages/apps/src/plugins/index.ts @@ -1 +1,2 @@ +export * from './feedback'; export * from './http';