[WIP] Add FeedbackPlugin for eval service integration

examples/ai/src/commands.ts

@@ -2,7 +2,7 @@ import { ActivityLike, IMessageActivity, SentActivity } from '@microsoft/teams.a
 
 import { OpenAIChatModel } from '@microsoft/teams.openai';
 
-import { ILogger } from '../../../packages/common/dist/logging/logger';
+import { ILogger } from '@microsoft/teams.common';
 
 import { handleFeedbackLoop } from './feedback';
 import { handleDocumentationSearch } from './simple-rag';

examples/ai/src/feedback.ts

@@ -5,18 +5,32 @@ import {
   MessageActivity,
   SentActivity,
 } from '@microsoft/teams.api';
+import { IFeedbackProvider, SentMessageData, FeedbackScore } from '@microsoft/teams.apps';
+import { ConsoleLogger } from '@microsoft/teams.common';
 
-// This store would ideally be persisted in a database
-export const storedFeedbackByMessageId = new Map<
-  string,
-  {
-    incomingMessage: string;
-    outgoingMessage: string;
-    likes: number;
-    dislikes: number;
-    feedbacks: string[];
+/**
+ * Simple in-memory feedback provider for demo purposes.
+ * In production, use a provider for LangSmith, LangFuse, Braintrust, or Azure AI Foundry.
+ */
+export class InMemoryFeedbackProvider implements IFeedbackProvider {
+  readonly messages = new Map<string, SentMessageData>();
+  readonly feedback = new Map<string, FeedbackScore[]>();
+  private readonly log = new ConsoleLogger('@tests/ai/feedback-provider');
+
+  async logSentMessage(data: SentMessageData): Promise<void> {
+    this.messages.set(data.messageId, data);
+    this.log.info(`Message logged for ${data.messageId}: input=${data.input}, output=${data.output}`);
+  }
+
+  async logFeedback(messageId: string, score: FeedbackScore): Promise<void> {
+    const existing = this.feedback.get(messageId) ?? [];
+    existing.push(score);
+    this.feedback.set(messageId, existing);
+    this.log.info(`Feedback logged for ${messageId}: ${score.reaction}`);
   }
->();
+}
+
+export const feedbackProvider = new InMemoryFeedbackProvider();
 
 export const handleFeedbackLoop = async (
   model: IChatModel,
@@ -31,22 +45,13 @@ export const handleFeedbackLoop = async (
   const result = await prompt.send(activity.text);
 
   if (result) {
-    const { id: sentMessageId } = await send(
+    await send(
       result.content != null
         ? new MessageActivity(result.content)
             .addAiGenerated()
             /** Add feedback buttons via this method */
             .addFeedback()
         : 'I did not generate a response.'
     );
-
-    storedFeedbackByMessageId.set(sentMessageId, {
-      incomingMessage: activity.text,
-      outgoingMessage: result.content ?? '',
-      likes: 0,
-      dislikes: 0,
-      feedbacks: [],
-    });
-
   }
 };

examples/ai/src/index.ts

@@ -1,6 +1,6 @@
 import { ChatPrompt } from '@microsoft/teams.ai';
 import { MessageActivity } from '@microsoft/teams.api';
-import { App } from '@microsoft/teams.apps';
+import { App, FeedbackPlugin } from '@microsoft/teams.apps';
 import { ConsoleLogger } from '@microsoft/teams.common';
 import { DevtoolsPlugin } from '@microsoft/teams.dev';
 import { OpenAIChatModel } from '@microsoft/teams.openai';
@@ -13,15 +13,15 @@ import {
   structuredOutputCommand,
   weatherCommand,
 } from './commands';
-import { storedFeedbackByMessageId } from './feedback';
+import { feedbackProvider } from './feedback';
 import { handleDocumentationSearch } from './simple-rag';
 import { handleStatefulConversation } from './stateful-prompts';
 
 const logger = new ConsoleLogger('@tests/ai');
 
 const app = new App({
   logger,
-  plugins: [new DevtoolsPlugin()],
+  plugins: [new DevtoolsPlugin(), new FeedbackPlugin({ provider: feedbackProvider })],
 });
 
 const model = new OpenAIChatModel({
@@ -144,27 +144,4 @@ app.on('message', async ({ send, activity, log }) => {
   await handleStatefulConversation(model, activity, send, log);
 });
 
-app.on('message.submit.feedback', async ({ activity, log }) => {
-  const { reaction, feedback: feedbackJson } = activity.value.actionValue;
-  if (activity.replyToId == null) {
-    log.warn(`No replyToId found for messageId ${activity.id}`);
-    return;
-  }
-  const existingFeedback = storedFeedbackByMessageId.get(activity.replyToId);
-  /**
-   * feedbackJson looks like:
-   * {"feedbackText":"Nice!"}
-   */
-  if (!existingFeedback) {
-    log.warn(`No feedback found for messageId ${activity.id}`);
-  } else {
-    storedFeedbackByMessageId.set(activity.id, {
-      ...existingFeedback,
-      likes: existingFeedback.likes + (reaction === 'like' ? 1 : 0),
-      dislikes: existingFeedback.dislikes + (reaction === 'dislike' ? 1 : 0),
-      feedbacks: [...existingFeedback.feedbacks, feedbackJson],
-    });
-  }
-});
-
 app.start(process.env.PORT || 3978).catch(console.error);

examples/ai/src/simple-rag.ts

@@ -3,7 +3,7 @@ import Fuse from 'fuse.js';
 import { ChatPrompt, IChatModel } from '@microsoft/teams.ai';
 import { ActivityLike, IMessageActivity, MessageActivity } from '@microsoft/teams.api';
 
-import { ILogger } from '../../../packages/common/dist/logging/logger';
+import { ILogger } from '@microsoft/teams.common';
 
 interface IDocumentationItem {
   id: string;

examples/ai/src/stateful-prompts.ts

@@ -1,7 +1,7 @@
 import { ChatPrompt, IChatModel, Message } from '@microsoft/teams.ai';
 import { ActivityLike, IMessageActivity, MessageActivity } from '@microsoft/teams.api';
 
-import { ILogger } from '../../../packages/common/dist/logging/logger';
+import { ILogger } from '@microsoft/teams.common';
 
 // Simple in-memory store for conversation histories
 // In your application, it may be a good idea to use a more

examples/ai/src/tool-calling.ts

@@ -1,7 +1,7 @@
 import { ChatPrompt, IChatModel } from '@microsoft/teams.ai';
 import { ActivityLike, IMessageActivity, SentActivity } from '@microsoft/teams.api';
 
-import { ILogger } from '../../../packages/common/dist/logging/logger';
+import { ILogger } from '@microsoft/teams.common';
 
 interface IPokemonSearch {
   pokemonName: string;

packages/apps/src/plugins/feedback/index.ts

@@ -0,0 +1,7 @@
+export { FeedbackPlugin } from './plugin';
+export type {
+  IFeedbackProvider,
+  FeedbackPluginOptions,
+  SentMessageData,
+  FeedbackScore,
+} from './types';