Skip to content

Display analyzed image in chat #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jck411 wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Display analyzed image in chat #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 148 additions & 94 deletions frontend/src/lib/stores/chat.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,46 @@ function attachmentsFromParts(
return attachments;
}

function toolFallbackText(
resultText: string | null,
status: string,
toolName: string,
): string {
if (typeof resultText === 'string' && resultText) {
return resultText;
}
if (status === 'started') {
return `Running ${toolName || 'tool'}…`;
}
if (status && toolName) {
return `Tool ${toolName} ${status}.`;
}
if (toolName) {
return `Tool ${toolName} responded.`;
}
return 'Tool response received.';
}

function buildToolMessageContent(
sessionId: string | null,
content: ChatMessageContent | null,
resultText: string | null,
status: string,
toolName: string,
): { content: ChatMessageContent; text: string; attachments: AttachmentResource[] } {
const fallback = toolFallbackText(resultText, status, toolName);
const resolvedContent: ChatMessageContent = content ?? fallback;
const normalized = normalizeMessageContent(resolvedContent);
const textValue = normalized.text || fallback || '';
const attachments =
normalized.parts.length > 0 ? attachmentsFromParts(normalized.parts, sessionId) : [];
return {
content: resolvedContent,
text: textValue,
attachments,
};
}

function mergeMessageContent(
existingContent: ChatMessageContent,
existingText: string,
Expand Down Expand Up @@ -528,106 +568,120 @@ function createChatStore() {
return { ...value, messages };
});
},
onToolEvent(payload) {
const callId = typeof payload.call_id === 'string' ? payload.call_id : createId('tool');
const status = typeof payload.status === 'string' ? payload.status : 'started';
const toolName = typeof payload.name === 'string' ? payload.name : 'tool';
const result = payload.result;
const toolResult =
typeof result === 'string'
? result
: result && typeof result === 'object'
? JSON.stringify(result)
onToolEvent(payload) {
const callId = typeof payload.call_id === 'string' ? payload.call_id : createId('tool');
const status = typeof payload.status === 'string' ? payload.status : 'started';
const toolName = typeof payload.name === 'string' ? payload.name : 'tool';
const result = payload.result;
const toolResult =
typeof result === 'string'
? result
: result && typeof result === 'object'
? JSON.stringify(result)
: null;
const rawContent = (payload as Record<string, unknown>).content;
const structuredContent =
typeof rawContent === 'string' || Array.isArray(rawContent)
? (rawContent as ChatMessageContent)
: null;
const serverMessageId =
typeof payload.message_id === 'number' ? payload.message_id : null;

let messageId = toolMessageIds.get(callId);
if (!messageId) {
messageId = createId('tool');
toolMessageIds.set(callId, messageId);
const fallbackCreatedAt = new Date().toISOString();
const createdAt = coalesceTimestamp(
payload.created_at,
payload.created_at_utc,
fallbackCreatedAt,
);
const createdAtUtc = coalesceTimestamp(
payload.created_at_utc,
createdAt ?? fallbackCreatedAt,
);
store.update((value) => ({
...value,
messages: [
...value.messages,
{
id: messageId as string,
role: 'tool',
content:
status === 'started'
? `Running ${toolName}…`
: toolResult ?? `Tool ${toolName} responded.`,
text:
status === 'started'
? `Running ${toolName}…`
: toolResult ?? `Tool ${toolName} responded.`,
attachments: [],
pending: status === 'started',
details: {
toolName,
toolStatus: status,
toolResult: toolResult ?? null,
serverMessageId,
},
createdAt,
createdAtUtc,
},
],
}));
} else {
store.update((value) => {
const messages = value.messages.map((message) => {
if (message.id !== messageId) {
return message;
}
const details = {
...(message.details ?? {}),
const serverMessageId =
typeof payload.message_id === 'number' ? payload.message_id : null;

let messageId = toolMessageIds.get(callId);
if (!messageId) {
messageId = createId('tool');
toolMessageIds.set(callId, messageId);
const fallbackCreatedAt = new Date().toISOString();
const createdAt = coalesceTimestamp(
payload.created_at,
payload.created_at_utc,
fallbackCreatedAt,
);
const createdAtUtc = coalesceTimestamp(
payload.created_at_utc,
createdAt ?? fallbackCreatedAt,
);
store.update((value) => {
const resolved = buildToolMessageContent(
value.sessionId,
structuredContent,
toolResult,
status,
toolName,
toolStatus: status,
toolResult: toolResult ?? (message.details?.toolResult ?? null),
serverMessageId:
serverMessageId ?? message.details?.serverMessageId ?? null,
};
const nextText =
toolResult ??
(status === 'started'
? `Running ${toolName}…`
: `Tool ${toolName} ${status}.`);
const nextCreatedAt = coalesceTimestamp(
payload.created_at,
payload.created_at_utc,
message.createdAt ?? null,
);
const nextCreatedAtUtc = coalesceTimestamp(
payload.created_at_utc,
message.createdAtUtc ?? null,
);
return {
...message,
content: nextText,
text: nextText,
attachments: message.attachments ?? [],
pending: status === 'started',
details,
createdAt: nextCreatedAt,
createdAtUtc:
nextCreatedAtUtc ?? nextCreatedAt ?? message.createdAtUtc ?? null,
...value,
messages: [
...value.messages,
{
id: messageId as string,
role: 'tool',
content: resolved.content,
text: resolved.text,
attachments: resolved.attachments,
pending: status === 'started',
details: {
toolName,
toolStatus: status,
toolResult: toolResult ?? null,
serverMessageId,
},
createdAt,
createdAtUtc,
},
],
};
});
return { ...value, messages };
});
}
},
} else {
store.update((value) => {
const messages = value.messages.map((message) => {
if (message.id !== messageId) {
return message;
}
const details = {
...(message.details ?? {}),
toolName,
toolStatus: status,
toolResult: toolResult ?? (message.details?.toolResult ?? null),
serverMessageId:
serverMessageId ?? message.details?.serverMessageId ?? null,
};
const resolved = buildToolMessageContent(
value.sessionId,
structuredContent,
toolResult,
status,
toolName,
);
const nextAttachments =
resolved.attachments.length > 0
? resolved.attachments
: message.attachments ?? [];
const nextCreatedAt = coalesceTimestamp(
payload.created_at,
payload.created_at_utc,
message.createdAt ?? null,
);
const nextCreatedAtUtc = coalesceTimestamp(
payload.created_at_utc,
message.createdAtUtc ?? null,
);
return {
...message,
content: resolved.content,
text: resolved.text,
attachments: nextAttachments,
pending: status === 'started',
details,
createdAt: nextCreatedAt,
createdAtUtc:
nextCreatedAtUtc ?? nextCreatedAt ?? message.createdAtUtc ?? null,
};
});
return { ...value, messages };
});
}
},
onDone() {
store.update((value) => {
const messages = value.messages.map((message) => {
Expand Down
42 changes: 29 additions & 13 deletions src/backend/chat/streaming/handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ async def stream_conversation(
candidate = request_metadata.get("client_assistant_message_id")
if isinstance(candidate, str):
assistant_client_message_id = candidate
settings = get_settings()
attachment_url_ttl = settings.attachment_signed_url_ttl
active_tools_payload = list(tools_payload)
tool_choice_value = request.tool_choice
requested_tool_choice = (
Expand Down Expand Up @@ -162,7 +164,7 @@ async def stream_conversation(
conversation_state = await refresh_message_attachments(
conversation_state,
self._repo,
ttl=get_settings().attachment_signed_url_ttl,
ttl=attachment_url_ttl,
)

payload = request.to_openrouter_payload(active_model)
Expand Down Expand Up @@ -786,6 +788,14 @@ async def stream_conversation(
"tool_call_id": tool_id,
"content": content_parts,
}

refreshed_tool_messages = await refresh_message_attachments(
[tool_message],
self._repo,
ttl=attachment_url_ttl,
)
if refreshed_tool_messages:
tool_message = refreshed_tool_messages[0]
else:
# Plain text result
tool_message = {
Expand All @@ -794,26 +804,32 @@ async def stream_conversation(
"content": result_text,
}

result_for_client = cleaned_text if attachment_ids else result_text

edt_iso, utc_iso = format_timestamp_for_client(tool_created_at)
if edt_iso is not None:
tool_message["created_at"] = edt_iso
if utc_iso is not None:
tool_message["created_at_utc"] = utc_iso
conversation_state.append(tool_message)

tool_event_payload = {
"status": status,
"name": tool_name,
"call_id": tool_id,
"result": result_for_client,
"message_id": tool_record_id,
"created_at": edt_iso or tool_created_at,
"created_at_utc": utc_iso or tool_created_at,
}
if attachment_ids:
tool_content_payload = tool_message.get("content")
if tool_content_payload is not None:
tool_event_payload["content"] = tool_content_payload

yield {
"event": "tool",
"data": json.dumps(
{
"status": status,
"name": tool_name,
"call_id": tool_id,
"result": result_text,
"message_id": tool_record_id,
"created_at": edt_iso or tool_created_at,
"created_at_utc": utc_iso or tool_created_at,
}
),
"data": json.dumps(tool_event_payload),
}

notice_reason = _classify_tool_followup(
Expand All @@ -827,7 +843,7 @@ async def stream_conversation(
"type": "tool_followup_required",
"tool": tool_name or "unknown",
"reason": notice_reason,
"message": result_text,
"message": result_for_client,
"attempt": hop_count,
"confirmation_required": True,
}
Expand Down