diff --git a/lib/agents/content/__tests__/extractMessageAttachments.test.ts b/lib/agents/content/__tests__/extractMessageAttachments.test.ts new file mode 100644 index 00000000..3db6fc38 --- /dev/null +++ b/lib/agents/content/__tests__/extractMessageAttachments.test.ts @@ -0,0 +1,275 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { extractMessageAttachments } from "../extractMessageAttachments"; + +vi.mock("@vercel/blob", () => ({ + put: vi.fn(), +})); + +const { put } = await import("@vercel/blob"); + +describe("extractMessageAttachments", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("returns null values when message has no attachments", async () => { + const message = { text: "hello", attachments: [] }; + const result = await extractMessageAttachments(message as never); + expect(result).toEqual({ songUrl: null, imageUrl: null }); + }); + + it("returns null values when attachments is undefined", async () => { + const message = { text: "hello" }; + const result = await extractMessageAttachments(message as never); + expect(result).toEqual({ songUrl: null, imageUrl: null }); + }); + + it("uploads audio with correct contentType", async () => { + const audioBuffer = Buffer.from("fake-audio-data"); + const message = { + text: "hello", + attachments: [ + { + type: "audio", + name: "my-song.mp3", + fetchData: vi.fn().mockResolvedValue(audioBuffer), + }, + ], + }; + vi.mocked(put).mockResolvedValue({ + url: "https://blob.vercel-storage.com/content-attachments/my-song.mp3", + } as never); + + const result = await extractMessageAttachments(message as never); + + expect(message.attachments[0].fetchData).toHaveBeenCalled(); + expect(put).toHaveBeenCalledWith(expect.stringContaining("my-song.mp3"), audioBuffer, { + access: "public", + contentType: "audio/mpeg", + }); + expect(result.songUrl).toBe("https://blob.vercel-storage.com/content-attachments/my-song.mp3"); + expect(result.imageUrl).toBeNull(); + }); + + it("extracts and uploads an image attachment", async () => { + const imageBuffer = Buffer.from("fake-image-data"); + const message = { + text: "hello", + attachments: [ + { + type: "image", + name: "face.png", + fetchData: vi.fn().mockResolvedValue(imageBuffer), + }, + ], + }; + vi.mocked(put).mockResolvedValue({ + url: "https://blob.vercel-storage.com/content-attachments/face.png", + } as never); + + const result = await extractMessageAttachments(message as never); + + expect(result.imageUrl).toBe("https://blob.vercel-storage.com/content-attachments/face.png"); + expect(result.songUrl).toBeNull(); + }); + + it("extracts both audio and image when both are attached", async () => { + const audioBuffer = Buffer.from("fake-audio"); + const imageBuffer = Buffer.from("fake-image"); + const message = { + text: "hello", + attachments: [ + { + type: "audio", + name: "song.mp3", + fetchData: vi.fn().mockResolvedValue(audioBuffer), + }, + { + type: "image", + name: "photo.jpg", + fetchData: vi.fn().mockResolvedValue(imageBuffer), + }, + ], + }; + vi.mocked(put).mockResolvedValueOnce({ + url: "https://blob.vercel-storage.com/song.mp3", + } as never); + vi.mocked(put).mockResolvedValueOnce({ + url: "https://blob.vercel-storage.com/photo.jpg", + } as never); + + const result = await extractMessageAttachments(message as never); + + expect(result.songUrl).toBe("https://blob.vercel-storage.com/song.mp3"); + expect(result.imageUrl).toBe("https://blob.vercel-storage.com/photo.jpg"); + }); + + it("uses attachment data buffer if fetchData is not available", async () => { + const audioBuffer = Buffer.from("inline-audio"); + const message = { + text: "hello", + attachments: [ + { + type: "audio", + name: "inline.mp3", + data: audioBuffer, + }, + ], + }; + vi.mocked(put).mockResolvedValue({ + url: "https://blob.vercel-storage.com/inline.mp3", + } as never); + + const result = await extractMessageAttachments(message as never); + + expect(put).toHaveBeenCalledWith(expect.stringContaining("inline.mp3"), audioBuffer, { + access: "public", + contentType: "audio/mpeg", + }); + expect(result.songUrl).toBe("https://blob.vercel-storage.com/inline.mp3"); + }); + + it("uses first audio and first image when multiple of same type exist", async () => { + const audio1 = Buffer.from("audio1"); + const audio2 = Buffer.from("audio2"); + const message = { + text: "hello", + attachments: [ + { type: "audio", name: "first.mp3", fetchData: vi.fn().mockResolvedValue(audio1) }, + { type: "audio", name: "second.mp3", fetchData: vi.fn().mockResolvedValue(audio2) }, + ], + }; + vi.mocked(put).mockResolvedValue({ + url: "https://blob.vercel-storage.com/first.mp3", + } as never); + + const result = await extractMessageAttachments(message as never); + + expect(put).toHaveBeenCalledTimes(1); + expect(result.songUrl).toBe("https://blob.vercel-storage.com/first.mp3"); + }); + + it("detects image from file type with image mimeType (Slack uploads)", async () => { + const imageBuffer = Buffer.from("fake-image"); + const message = { + text: "hello", + attachments: [ + { + type: "file", + name: "photo.jpg", + mimeType: "image/jpeg", + fetchData: vi.fn().mockResolvedValue(imageBuffer), + }, + ], + }; + vi.mocked(put).mockResolvedValue({ + url: "https://blob.vercel-storage.com/photo.jpg", + } as never); + + const result = await extractMessageAttachments(message as never); + + expect(result.imageUrl).toBe("https://blob.vercel-storage.com/photo.jpg"); + }); + + it("detects audio from file type with audio mimeType (Slack uploads)", async () => { + const audioBuffer = Buffer.from("fake-audio"); + const message = { + text: "hello", + attachments: [ + { + type: "file", + name: "song.mp3", + mimeType: "audio/mpeg", + fetchData: vi.fn().mockResolvedValue(audioBuffer), + }, + ], + }; + vi.mocked(put).mockResolvedValue({ + url: "https://blob.vercel-storage.com/song.mp3", + } as never); + + const result = await extractMessageAttachments(message as never); + + expect(result.songUrl).toBe("https://blob.vercel-storage.com/song.mp3"); + }); + + it("ignores file attachments that are not audio or image", async () => { + const message = { + text: "hello", + attachments: [ + { type: "file", name: "document.pdf", fetchData: vi.fn() }, + { type: "video", name: "clip.mp4", fetchData: vi.fn() }, + ], + }; + + const result = await extractMessageAttachments(message as never); + + expect(put).not.toHaveBeenCalled(); + expect(result).toEqual({ songUrl: null, imageUrl: null }); + }); + + it("returns null when attachment has no data and no fetchData", async () => { + const message = { + text: "hello", + attachments: [ + { + type: "audio", + name: "empty.mp3", + }, + ], + }; + + const result = await extractMessageAttachments(message as never); + + expect(put).not.toHaveBeenCalled(); + expect(result.songUrl).toBeNull(); + }); + + it("gracefully handles upload failure without crashing", async () => { + const audioBuffer = Buffer.from("fake-audio"); + const imageBuffer = Buffer.from("fake-image"); + const message = { + text: "hello", + attachments: [ + { + type: "audio", + name: "song.mp3", + fetchData: vi.fn().mockResolvedValue(audioBuffer), + }, + { + type: "image", + name: "photo.jpg", + fetchData: vi.fn().mockResolvedValue(imageBuffer), + }, + ], + }; + // First call (audio) throws, second call (image) succeeds + vi.mocked(put).mockRejectedValueOnce(new Error("upload failed")); + vi.mocked(put).mockResolvedValueOnce({ + url: "https://blob.vercel-storage.com/photo.jpg", + } as never); + + const result = await extractMessageAttachments(message as never); + + expect(result.songUrl).toBeNull(); + expect(result.imageUrl).toBe("https://blob.vercel-storage.com/photo.jpg"); + }); + + it("falls back to generic name when attachment name is missing", async () => { + const audioBuffer = Buffer.from("audio"); + const message = { + text: "hello", + attachments: [{ type: "audio", fetchData: vi.fn().mockResolvedValue(audioBuffer) }], + }; + vi.mocked(put).mockResolvedValue({ + url: "https://blob.vercel-storage.com/attachment", + } as never); + + await extractMessageAttachments(message as never); + + expect(put).toHaveBeenCalledWith(expect.stringContaining("attachment"), audioBuffer, { + access: "public", + contentType: "audio/mpeg", + }); + }); +}); diff --git a/lib/agents/content/__tests__/registerOnNewMention.test.ts b/lib/agents/content/__tests__/registerOnNewMention.test.ts index 9f1a147c..aedd578d 100644 --- a/lib/agents/content/__tests__/registerOnNewMention.test.ts +++ b/lib/agents/content/__tests__/registerOnNewMention.test.ts @@ -29,11 +29,16 @@ vi.mock("../parseContentPrompt", () => ({ parseContentPrompt: vi.fn(), })); +vi.mock("../extractMessageAttachments", () => ({ + extractMessageAttachments: vi.fn(), +})); + const { triggerCreateContent } = await import("@/lib/trigger/triggerCreateContent"); const { triggerPollContentRun } = await import("@/lib/trigger/triggerPollContentRun"); const { resolveArtistSlug } = await import("@/lib/content/resolveArtistSlug"); const { getArtistContentReadiness } = await import("@/lib/content/getArtistContentReadiness"); const { parseContentPrompt } = await import("../parseContentPrompt"); +const { extractMessageAttachments } = await import("../extractMessageAttachments"); /** * Creates a mock content agent bot for testing. @@ -76,6 +81,10 @@ function createMockMessage(text: string) { describe("registerOnNewMention", () => { beforeEach(() => { vi.clearAllMocks(); + vi.mocked(extractMessageAttachments).mockResolvedValue({ + songUrl: null, + imageUrl: null, + }); }); it("registers a handler on the bot", () => { @@ -273,6 +282,126 @@ describe("registerOnNewMention", () => { expect(ackMessage).toContain("Videos:"); }); + it("adds song URL to songs array when audio is attached", async () => { + const bot = createMockBot(); + registerOnNewMention(bot as never); + + vi.mocked(parseContentPrompt).mockResolvedValue({ + lipsync: false, + batch: 1, + captionLength: "short", + upscale: false, + template: "artist-caption-bedroom", + }); + vi.mocked(resolveArtistSlug).mockResolvedValue("test-artist"); + vi.mocked(getArtistContentReadiness).mockResolvedValue({ + githubRepo: "https://github.com/test/repo", + } as never); + vi.mocked(extractMessageAttachments).mockResolvedValue({ + songUrl: "https://blob.vercel-storage.com/song.mp3", + imageUrl: null, + }); + vi.mocked(triggerCreateContent).mockResolvedValue({ id: "run-1" }); + vi.mocked(triggerPollContentRun).mockResolvedValue(undefined as never); + + const thread = createMockThread(); + const message = createMockMessage("make a video"); + await bot.getHandler()!(thread, message); + + const payload = vi.mocked(triggerCreateContent).mock.calls[0][0]; + expect(payload.songs).toContain("https://blob.vercel-storage.com/song.mp3"); + }); + + it("passes images array to triggerCreateContent when image is attached", async () => { + const bot = createMockBot(); + registerOnNewMention(bot as never); + + vi.mocked(parseContentPrompt).mockResolvedValue({ + lipsync: false, + batch: 1, + captionLength: "short", + upscale: false, + template: "artist-caption-bedroom", + }); + vi.mocked(resolveArtistSlug).mockResolvedValue("test-artist"); + vi.mocked(getArtistContentReadiness).mockResolvedValue({ + githubRepo: "https://github.com/test/repo", + } as never); + vi.mocked(extractMessageAttachments).mockResolvedValue({ + songUrl: null, + imageUrl: "https://blob.vercel-storage.com/face.png", + }); + vi.mocked(triggerCreateContent).mockResolvedValue({ id: "run-1" }); + vi.mocked(triggerPollContentRun).mockResolvedValue(undefined as never); + + const thread = createMockThread(); + const message = createMockMessage("make a video"); + await bot.getHandler()!(thread, message); + + expect(triggerCreateContent).toHaveBeenCalledWith( + expect.objectContaining({ + images: ["https://blob.vercel-storage.com/face.png"], + }), + ); + }); + + it("omits images from payload when no media is attached", async () => { + const bot = createMockBot(); + registerOnNewMention(bot as never); + + vi.mocked(parseContentPrompt).mockResolvedValue({ + lipsync: false, + batch: 1, + captionLength: "short", + upscale: false, + template: "artist-caption-bedroom", + }); + vi.mocked(resolveArtistSlug).mockResolvedValue("test-artist"); + vi.mocked(getArtistContentReadiness).mockResolvedValue({ + githubRepo: "https://github.com/test/repo", + } as never); + vi.mocked(triggerCreateContent).mockResolvedValue({ id: "run-1" }); + vi.mocked(triggerPollContentRun).mockResolvedValue(undefined as never); + + const thread = createMockThread(); + const message = createMockMessage("make a video"); + await bot.getHandler()!(thread, message); + + const payload = vi.mocked(triggerCreateContent).mock.calls[0][0]; + expect(payload).not.toHaveProperty("images"); + }); + + it("includes attached media notes in acknowledgment message", async () => { + const bot = createMockBot(); + registerOnNewMention(bot as never); + + vi.mocked(parseContentPrompt).mockResolvedValue({ + lipsync: false, + batch: 1, + captionLength: "short", + upscale: false, + template: "artist-caption-bedroom", + }); + vi.mocked(resolveArtistSlug).mockResolvedValue("test-artist"); + vi.mocked(getArtistContentReadiness).mockResolvedValue({ + githubRepo: "https://github.com/test/repo", + } as never); + vi.mocked(extractMessageAttachments).mockResolvedValue({ + songUrl: "https://blob.vercel-storage.com/song.mp3", + imageUrl: "https://blob.vercel-storage.com/face.png", + }); + vi.mocked(triggerCreateContent).mockResolvedValue({ id: "run-1" }); + vi.mocked(triggerPollContentRun).mockResolvedValue(undefined as never); + + const thread = createMockThread(); + const message = createMockMessage("make a video"); + await bot.getHandler()!(thread, message); + + const ackMessage = thread.post.mock.calls[0][0] as string; + expect(ackMessage).toContain("Audio: attached file"); + expect(ackMessage).toContain("Image: attached file"); + }); + it("includes song names in acknowledgment message", async () => { const bot = createMockBot(); registerOnNewMention(bot as never); diff --git a/lib/agents/content/extractMessageAttachments.ts b/lib/agents/content/extractMessageAttachments.ts new file mode 100644 index 00000000..de40806f --- /dev/null +++ b/lib/agents/content/extractMessageAttachments.ts @@ -0,0 +1,63 @@ +import { resolveAttachmentUrl } from "./resolveAttachmentUrl"; + +interface Attachment { + type: "image" | "file" | "video" | "audio"; + mimeType?: string; + name?: string; + url?: string; + data?: Buffer | Blob; + fetchData?: () => Promise; +} + +interface MessageWithAttachments { + attachments?: Attachment[]; +} + +export interface ExtractedAttachments { + songUrl: string | null; + imageUrl: string | null; +} + +/** + * Extracts audio and image attachments from a Slack message and returns + * public URLs for the content pipeline. + * + * @param message + */ +export async function extractMessageAttachments( + message: MessageWithAttachments, +): Promise { + const result: ExtractedAttachments = { + songUrl: null, + imageUrl: null, + }; + + const attachments = message.attachments; + if (!attachments || attachments.length === 0) { + return result; + } + + const isAudio = (a: Attachment) => a.type === "audio" || a.mimeType?.startsWith("audio/"); + const isImage = (a: Attachment) => a.type === "image" || a.mimeType?.startsWith("image/"); + + const audioAttachment = attachments.find(isAudio); + const imageAttachment = attachments.find(isImage); + + if (audioAttachment) { + try { + result.songUrl = await resolveAttachmentUrl(audioAttachment, "audio"); + } catch (error) { + console.error("[content-agent] Failed to resolve audio attachment:", error); + } + } + + if (imageAttachment) { + try { + result.imageUrl = await resolveAttachmentUrl(imageAttachment, "image"); + } catch (error) { + console.error("[content-agent] Failed to resolve image attachment:", error); + } + } + + return result; +} diff --git a/lib/agents/content/handlers/registerOnNewMention.ts b/lib/agents/content/handlers/registerOnNewMention.ts index d5e4b706..09236116 100644 --- a/lib/agents/content/handlers/registerOnNewMention.ts +++ b/lib/agents/content/handlers/registerOnNewMention.ts @@ -5,6 +5,7 @@ import { resolveArtistSlug } from "@/lib/content/resolveArtistSlug"; import { getArtistContentReadiness } from "@/lib/content/getArtistContentReadiness"; import { selectAccountSnapshots } from "@/lib/supabase/account_snapshots/selectAccountSnapshots"; import { parseContentPrompt } from "../parseContentPrompt"; +import { extractMessageAttachments } from "../extractMessageAttachments"; /** * Registers the onNewMention handler on the content agent bot. @@ -25,6 +26,9 @@ export function registerOnNewMention(bot: ContentAgentBot) { message.text, ); + // Extract audio/image attachments from the Slack message + const { songUrl, imageUrl } = await extractMessageAttachments(message); + // Resolve artist slug const artistSlug = await resolveArtistSlug(artistAccountId); if (!artistSlug) { @@ -65,10 +69,19 @@ export function registerOnNewMention(bot: ContentAgentBot) { if (songs && songs.length > 0) { details.push(`- Songs: ${songs.join(", ")}`); } + if (songUrl) { + details.push("- Audio: attached file"); + } + if (imageUrl) { + details.push("- Image: attached file (face guide)"); + } await thread.post( `Generating content...\n${details.join("\n")}\n\nI'll reply here when ready (~5-10 min).`, ); + // Build songs array: merge parsed slugs with attached audio URL + const allSongs = [...(songs ?? []), ...(songUrl ? [songUrl] : [])]; + // Trigger content creation const payload = { accountId, @@ -78,7 +91,8 @@ export function registerOnNewMention(bot: ContentAgentBot) { captionLength, upscale, githubRepo, - ...(songs && songs.length > 0 && { songs }), + ...(allSongs.length > 0 && { songs: allSongs }), + ...(imageUrl && { images: [imageUrl] }), }; const results = await Promise.allSettled( diff --git a/lib/agents/content/resolveAttachmentUrl.ts b/lib/agents/content/resolveAttachmentUrl.ts new file mode 100644 index 00000000..6293342f --- /dev/null +++ b/lib/agents/content/resolveAttachmentUrl.ts @@ -0,0 +1,57 @@ +import { put } from "@vercel/blob"; +import { downloadSlackFile } from "@/lib/slack/downloadSlackFile"; +import { extractSlackFileId } from "@/lib/slack/extractSlackFileId"; + +interface Attachment { + type: "image" | "file" | "video" | "audio"; + mimeType?: string; + name?: string; + url?: string; + data?: Buffer | Blob; + fetchData?: () => Promise; +} + +/** + * Downloads a Slack file and uploads to Vercel Blob. + * + * Uses Slack's files.info API to get the url_private_download URL, + * then downloads with Bearer token auth. The attachment.url from the + * Chat SDK is a thumbnail URL that doesn't serve actual file content. + * + * @param attachment + * @param prefix + */ +export async function resolveAttachmentUrl( + attachment: Attachment, + prefix: string, +): Promise { + let data: Buffer | null = null; + const token = process.env.SLACK_CONTENT_BOT_TOKEN; + + if (attachment.url && token) { + const fileId = extractSlackFileId(attachment.url); + if (fileId) { + data = await downloadSlackFile(fileId, token); + } + } + + // Fallback to fetchData / data + if (!data) { + const raw = attachment.fetchData ? await attachment.fetchData() : attachment.data; + if (raw) data = Buffer.isBuffer(raw) ? raw : Buffer.from(raw as unknown as ArrayBuffer); + } + + if (!data) { + console.error( + `[content-agent] Could not download attachment "${attachment.name ?? "unknown"}"`, + ); + return null; + } + + const filename = attachment.name ?? "attachment"; + const blobPath = `content-attachments/${prefix}/${Date.now()}-${filename}`; + const contentType = attachment.mimeType ?? (prefix === "audio" ? "audio/mpeg" : "image/png"); + + const blob = await put(blobPath, data, { access: "public", contentType }); + return blob.url; +} diff --git a/lib/content/createContentHandler.ts b/lib/content/createContentHandler.ts index 5ee90515..577e8132 100644 --- a/lib/content/createContentHandler.ts +++ b/lib/content/createContentHandler.ts @@ -9,6 +9,8 @@ import { selectAccountSnapshots } from "@/lib/supabase/account_snapshots/selectA /** * Handler for POST /api/content/create. * Always returns runIds array (KISS — one response shape for single and batch). + * + * @param request */ export async function createContentHandler(request: NextRequest): Promise { const validated = await validateCreateContentBody(request); @@ -50,6 +52,7 @@ export async function createContentHandler(request: NextRequest): Promise { + const infoResponse = await fetch(`https://slack.com/api/files.info?file=${fileId}`, { + headers: { Authorization: `Bearer ${token}` }, + }); + + if (!infoResponse.ok) { + console.error(`[content-agent] files.info failed: ${infoResponse.status}`); + return null; + } + + const info = (await infoResponse.json()) as { + ok: boolean; + file?: { url_private_download?: string; url_private?: string; size?: number }; + error?: string; + }; + + if (!info.ok || !info.file) { + console.error(`[content-agent] files.info error: ${info.error ?? "no file"}`); + return null; + } + + const downloadUrl = info.file.url_private_download ?? info.file.url_private; + if (!downloadUrl) { + console.error(`[content-agent] No download URL in files.info response`); + return null; + } + + const fileResponse = await fetch(downloadUrl, { + headers: { Authorization: `Bearer ${token}` }, + }); + + if (!fileResponse.ok) { + console.error(`[content-agent] File download failed: ${fileResponse.status}`); + return null; + } + + const contentType = fileResponse.headers.get("content-type") ?? ""; + if (contentType.includes("text/html")) { + console.error(`[content-agent] Got HTML instead of file content`); + return null; + } + + return Buffer.from(await fileResponse.arrayBuffer()); +} diff --git a/lib/slack/extractSlackFileId.ts b/lib/slack/extractSlackFileId.ts new file mode 100644 index 00000000..0c840d98 --- /dev/null +++ b/lib/slack/extractSlackFileId.ts @@ -0,0 +1,22 @@ +/** + * Extracts the Slack file ID (e.g. F0APMKTKG9M) from a Slack file URL. + * URL format: files-tmb/TEAMID-FILEID-HASH/name or files-pri/TEAMID-FILEID/name + * + * @param url + */ +export function extractSlackFileId(url: string): string | null { + try { + const parts = new URL(url).pathname.split("/").filter(Boolean); + // parts[1] is "TEAMID-FILEID-HASH" or "TEAMID-FILEID" + if (parts.length >= 2) { + const segments = parts[1].split("-"); + // File ID is the second segment (e.g. F0APMKTKG9M) + if (segments.length >= 2) { + return segments[1]; + } + } + } catch { + // ignore + } + return null; +} diff --git a/lib/trigger/triggerCreateContent.ts b/lib/trigger/triggerCreateContent.ts index c40bd53d..9d6e5bd8 100644 --- a/lib/trigger/triggerCreateContent.ts +++ b/lib/trigger/triggerCreateContent.ts @@ -12,12 +12,16 @@ export interface TriggerCreateContentPayload { upscale: boolean; /** GitHub repo URL so the task can fetch artist files. */ githubRepo: string; - /** Optional list of song slugs to restrict which songs the pipeline picks from. */ + /** Optional list of song slugs or public URLs. URLs are downloaded directly by the task. */ songs?: string[]; + /** Optional list of public image URLs to use as face guides. */ + images?: string[]; } /** * Triggers the create-content task in Trigger.dev. + * + * @param payload */ export async function triggerCreateContent(payload: TriggerCreateContentPayload) { const handle = await tasks.trigger(CREATE_CONTENT_TASK_ID, payload);