From 60b4c3f11505ebddf436939173692ee7df03229c Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Tue, 17 Mar 2026 23:31:32 -0400 Subject: [PATCH 01/14] Add on-device AI assistant powered by MLX for content creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrates mlx-swift-examples v2 into MuseCore for local LLM inference, with a streaming chat UI in ArkavoCreator that helps creators draft, rewrite, and adapt content across platforms (Bluesky, YouTube, Twitch, Reddit, Micro.blog) — no backend required. MuseCore additions: - StreamingLLMProvider protocol for async token streams - MLXBackend wrapping MLXLMCommon generate API - ModelRegistry catalog (Gemma 270M default, Qwen 3.5 scale-up) - ModelManager for lifecycle, memory budgeting, GPU coexistence ArkavoCreator additions: - PlatformContext protocol with per-platform constraints/actions - AssistantPromptBuilder for context-aware system prompts - AssistantChatView (full section) and AssistantPanelView (Cmd+Shift+A) - AssistantViewModel with streaming generation and auto-unload in Studio - localAssistant feature flag (ships enabled, independent of aiAgent) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../xcshareddata/swiftpm/Package.resolved | 63 ++++ .../ArkavoCreator/ArkavoCreatorApp.swift | 12 +- .../Assistant/AssistantChatView.swift | 273 ++++++++++++++++++ .../Assistant/AssistantPanelView.swift | 157 ++++++++++ .../Assistant/AssistantPromptBuilder.swift | 88 ++++++ .../Assistant/AssistantViewModel.swift | 148 ++++++++++ .../Assistant/PlatformContext.swift | 135 +++++++++ ArkavoCreator/ArkavoCreator/ContentView.swift | 37 ++- .../ArkavoCreator/FeatureFlags.swift | 2 + MuseCore/Package.resolved | 65 ++++- MuseCore/Package.swift | 9 +- .../Sources/MuseCore/LLM/MLXBackend.swift | 125 ++++++++ .../Sources/MuseCore/LLM/ModelManager.swift | 94 ++++++ .../Sources/MuseCore/LLM/ModelRegistry.swift | 81 ++++++ .../MuseCore/LLM/StreamingLLMProvider.swift | 47 +++ 15 files changed, 1327 insertions(+), 9 deletions(-) create mode 100644 ArkavoCreator/ArkavoCreator/Assistant/AssistantChatView.swift create mode 100644 ArkavoCreator/ArkavoCreator/Assistant/AssistantPanelView.swift create mode 100644 ArkavoCreator/ArkavoCreator/Assistant/AssistantPromptBuilder.swift create mode 100644 ArkavoCreator/ArkavoCreator/Assistant/AssistantViewModel.swift create mode 100644 ArkavoCreator/ArkavoCreator/Assistant/PlatformContext.swift create mode 100644 MuseCore/Sources/MuseCore/LLM/MLXBackend.swift create mode 100644 MuseCore/Sources/MuseCore/LLM/ModelManager.swift create mode 100644 MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift create mode 100644 MuseCore/Sources/MuseCore/LLM/StreamingLLMProvider.swift diff --git a/ArkavoCreator/ArkavoCreator.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/ArkavoCreator/ArkavoCreator.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index f4545fdd..59fdbeb8 100644 --- a/ArkavoCreator/ArkavoCreator.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/ArkavoCreator/ArkavoCreator.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -10,6 +10,15 @@ "version" : "1.9.0" } }, + { + "identity" : "gzipswift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/1024jp/GzipSwift", + "state" : { + "revision" : "731037f6cc2be2ec01562f6597c1d0aa3fe6fd05", + "version" : "6.0.1" + } + }, { "identity" : "iroh-swift", "kind" : "remoteSourceControl", @@ -19,6 +28,24 @@ "version" : "0.3.0" } }, + { + "identity" : "mlx-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift", + "state" : { + "revision" : "072b684acaae80b6a463abab3a103732f33774bf", + "version" : "0.29.1" + } + }, + { + "identity" : "mlx-swift-examples", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift-examples", + "state" : { + "revision" : "9bff95ca5f0b9e8c021acc4d71a2bbe4a7441631", + "version" : "2.29.1" + } + }, { "identity" : "opentdfkit", "kind" : "remoteSourceControl", @@ -28,6 +55,42 @@ "revision" : "d8ffeff99e00ec3334aa57b1fe5f9e1c7f38d2a9" } }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "8d9834a6189db730f6264db7556a7ffb751e99ee", + "version" : "1.4.0" + } + }, + { + "identity" : "swift-jinja", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-jinja.git", + "state" : { + "revision" : "f731f03bf746481d4fda07f817c3774390c4d5b9", + "version" : "2.3.2" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-transformers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-transformers", + "state" : { + "revision" : "a2e184dddb4757bc943e77fbe99ac6786c53f0b2", + "version" : "1.0.0" + } + }, { "identity" : "vrmmetalkit", "kind" : "remoteSourceControl", diff --git a/ArkavoCreator/ArkavoCreator/ArkavoCreatorApp.swift b/ArkavoCreator/ArkavoCreator/ArkavoCreatorApp.swift index 469e9982..dd109d2f 100644 --- a/ArkavoCreator/ArkavoCreator/ArkavoCreatorApp.swift +++ b/ArkavoCreator/ArkavoCreator/ArkavoCreatorApp.swift @@ -2,6 +2,7 @@ import ArkavoKit import ArkavoSocial import AuthenticationServices import LocalAuthentication +import MuseCore import SwiftData import SwiftUI @@ -18,6 +19,9 @@ struct ArkavoCreatorApp: App { @StateObject private var windowAccessor = WindowAccessor.shared @StateObject private var agentService = CreatorAgentService() + @State private var modelManager = ModelManager() + @State private var assistantViewModel: AssistantViewModel? + let patreonClient = PatreonClient(clientId: Secrets.patreonClientId, clientSecret: Secrets.patreonClientSecret) let redditClient = RedditClient(clientId: Secrets.redditClientId) let micropubClient = MicropubClient(clientId: Config.micropubClientID) @@ -53,9 +57,15 @@ struct ArkavoCreatorApp: App { micropubClient: micropubClient, blueskyClient: blueskyClient, youtubeClient: youtubeClient, - agentService: agentService + agentService: agentService, + assistantViewModel: assistantViewModel ?? AssistantViewModel(modelManager: modelManager) ) .onAppear { + // Initialize assistant view model + if assistantViewModel == nil { + assistantViewModel = AssistantViewModel(modelManager: modelManager) + } + // Load stored tokens redditClient.loadStoredTokens() micropubClient.loadStoredTokens() diff --git a/ArkavoCreator/ArkavoCreator/Assistant/AssistantChatView.swift b/ArkavoCreator/ArkavoCreator/Assistant/AssistantChatView.swift new file mode 100644 index 00000000..19c81bea --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Assistant/AssistantChatView.swift @@ -0,0 +1,273 @@ +import MuseCore +import SwiftUI + +/// Full AI assistant chat view for the Assistant section +struct AssistantChatView: View { + @Bindable var viewModel: AssistantViewModel + + @State private var inputText = "" + @FocusState private var isInputFocused: Bool + + var body: some View { + VStack(spacing: 0) { + // Model status bar + modelStatusBar + + // Platform context indicator + platformContextBar + + Divider() + + // Messages + ScrollViewReader { proxy in + ScrollView { + LazyVStack(alignment: .leading, spacing: 12) { + ForEach(viewModel.messages) { message in + MessageBubble(message: message) + .id(message.id) + } + + // Streaming indicator + if viewModel.isGenerating, !viewModel.streamingText.isEmpty { + MessageBubble( + message: AssistantMessage(role: .assistant, content: viewModel.streamingText) + ) + .id("streaming") + } + } + .padding() + } + .onChange(of: viewModel.messages.count) { + withAnimation { + if let lastID = viewModel.messages.last?.id { + proxy.scrollTo(lastID, anchor: .bottom) + } + } + } + .onChange(of: viewModel.streamingText) { + if viewModel.isGenerating { + proxy.scrollTo("streaming", anchor: .bottom) + } + } + } + + Divider() + + // Quick actions + quickActionsBar + + // Input bar + inputBar + } + } + + // MARK: - Components + + private var modelStatusBar: some View { + HStack(spacing: 8) { + Circle() + .fill(statusColor) + .frame(width: 8, height: 8) + + Text(statusText) + .font(.caption) + .foregroundStyle(.secondary) + + Spacer() + + // Model picker + Menu { + ForEach(viewModel.modelManager.availableModels) { model in + Button { + Task { await viewModel.modelManager.selectModel(model) } + } label: { + HStack { + Text(model.displayName) + if model == viewModel.modelManager.selectedModel { + Image(systemName: "checkmark") + } + } + } + } + } label: { + HStack(spacing: 4) { + Text(viewModel.modelManager.selectedModel.displayName) + .font(.caption) + Image(systemName: "chevron.down") + .font(.caption2) + } + .foregroundStyle(.secondary) + } + .menuStyle(.borderlessButton) + .fixedSize() + + // Load/unload button + if viewModel.modelManager.state == .idle || viewModel.modelManager.state == .error("") || isUnloaded { + Button("Load") { + Task { await viewModel.modelManager.loadSelectedModel() } + } + .buttonStyle(.bordered) + .controlSize(.small) + } + } + .padding(.horizontal) + .padding(.vertical, 8) + .background(.bar) + } + + private var isUnloaded: Bool { + if case .unloaded = viewModel.modelManager.state { return true } + return false + } + + private var statusColor: Color { + switch viewModel.modelManager.state { + case .ready: .green + case .loading, .downloading: .orange + case .error: .red + default: .gray + } + } + + private var statusText: String { + switch viewModel.modelManager.state { + case .idle: "Model not loaded" + case .downloading(let progress): "Downloading \(Int(progress * 100))%" + case .loading: "Loading model..." + case .ready: "Ready" + case .error(let msg): "Error: \(msg)" + case .unloaded(let reason): reason + } + } + + private var platformContextBar: some View { + let context = viewModel.modelManager.selectedModel.displayName + return HStack(spacing: 8) { + Image(systemName: "sparkles") + .font(.caption) + .foregroundStyle(.secondary) + Text("Local inference with \(context)") + .font(.caption) + .foregroundStyle(.secondary) + Spacer() + } + .padding(.horizontal) + .padding(.vertical, 4) + } + + private var quickActionsBar: some View { + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 8) { + ForEach(currentActions, id: \.self) { action in + Button(action.rawValue) { + viewModel.performAction(action) + } + .buttonStyle(.bordered) + .controlSize(.small) + .disabled(!viewModel.modelManager.isReady || viewModel.isGenerating) + } + } + .padding(.horizontal) + .padding(.vertical, 6) + } + } + + private var currentActions: [AssistantAction] { + // Get actions from the current navigation context + GenericContext().suggestedActions + } + + private var inputBar: some View { + HStack(spacing: 8) { + TextField("Ask the assistant...", text: $inputText, axis: .vertical) + .textFieldStyle(.plain) + .lineLimit(1...5) + .focused($isInputFocused) + .onSubmit { + sendMessage() + } + + if viewModel.isGenerating { + Button { + viewModel.stopGeneration() + } label: { + Image(systemName: "stop.circle.fill") + .foregroundStyle(.red) + } + .buttonStyle(.plain) + } else { + Button { + sendMessage() + } label: { + Image(systemName: "arrow.up.circle.fill") + .foregroundStyle(inputText.isEmpty ? .gray : .accentColor) + } + .buttonStyle(.plain) + .disabled(inputText.isEmpty) + } + } + .padding() + } + + private func sendMessage() { + let text = inputText + inputText = "" + viewModel.send(message: text) + } +} + +// MARK: - Message Bubble + +private struct MessageBubble: View { + let message: AssistantMessage + + var body: some View { + HStack(alignment: .top, spacing: 10) { + avatar + + VStack(alignment: .leading, spacing: 4) { + Text(message.content) + .textSelection(.enabled) + .font(message.role == .system ? .caption : .body) + .foregroundStyle(message.role == .system ? .secondary : .primary) + + Text(message.timestamp, style: .time) + .font(.caption2) + .foregroundStyle(.tertiary) + } + + Spacer(minLength: 40) + + if message.role == .assistant { + Button { + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(message.content, forType: .string) + } label: { + Image(systemName: "doc.on.doc") + .font(.caption) + .foregroundStyle(.secondary) + } + .buttonStyle(.plain) + } + } + .padding(.horizontal, 4) + } + + @ViewBuilder + private var avatar: some View { + switch message.role { + case .user: + Image(systemName: "person.circle.fill") + .foregroundStyle(.blue) + .font(.title3) + case .assistant: + Image(systemName: "sparkles") + .foregroundStyle(.purple) + .font(.title3) + case .system: + Image(systemName: "info.circle") + .foregroundStyle(.secondary) + .font(.title3) + } + } +} diff --git a/ArkavoCreator/ArkavoCreator/Assistant/AssistantPanelView.swift b/ArkavoCreator/ArkavoCreator/Assistant/AssistantPanelView.swift new file mode 100644 index 00000000..6ee56841 --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Assistant/AssistantPanelView.swift @@ -0,0 +1,157 @@ +import MuseCore +import SwiftUI + +/// Compact floating panel for quick assistant access +struct AssistantPanelView: View { + @Bindable var viewModel: AssistantViewModel + + @State private var inputText = "" + @FocusState private var isInputFocused: Bool + @Environment(\.dismiss) private var dismiss + + var body: some View { + VStack(spacing: 0) { + // Header + HStack { + Label("AI Assistant", systemImage: "sparkles") + .font(.headline) + Spacer() + + // Model status + HStack(spacing: 4) { + Circle() + .fill(viewModel.modelManager.isReady ? .green : .gray) + .frame(width: 6, height: 6) + Text(viewModel.modelManager.selectedModel.displayName) + .font(.caption) + .foregroundStyle(.secondary) + } + + Button { + dismiss() + } label: { + Image(systemName: "xmark.circle.fill") + .foregroundStyle(.secondary) + } + .buttonStyle(.plain) + } + .padding() + + Divider() + + // Recent messages (last 5) + ScrollViewReader { proxy in + ScrollView { + LazyVStack(alignment: .leading, spacing: 8) { + ForEach(viewModel.messages.suffix(5)) { message in + PanelMessageRow(message: message) + .id(message.id) + } + + if viewModel.isGenerating, !viewModel.streamingText.isEmpty { + PanelMessageRow( + message: AssistantMessage(role: .assistant, content: viewModel.streamingText) + ) + .id("streaming") + } + } + .padding(.horizontal) + .padding(.vertical, 8) + } + .onChange(of: viewModel.messages.count) { + if let lastID = viewModel.messages.last?.id { + proxy.scrollTo(lastID, anchor: .bottom) + } + } + } + + Divider() + + // Quick actions + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 6) { + ForEach(AssistantAction.allCases.prefix(4), id: \.self) { action in + Button(action.rawValue) { + viewModel.performAction(action) + } + .buttonStyle(.bordered) + .controlSize(.mini) + .disabled(!viewModel.modelManager.isReady || viewModel.isGenerating) + } + } + .padding(.horizontal) + .padding(.vertical, 6) + } + + // Input + HStack(spacing: 8) { + TextField("Ask something...", text: $inputText) + .textFieldStyle(.plain) + .focused($isInputFocused) + .onSubmit { sendMessage() } + + if viewModel.isGenerating { + Button { + viewModel.stopGeneration() + } label: { + Image(systemName: "stop.circle.fill") + .foregroundStyle(.red) + } + .buttonStyle(.plain) + } else { + Button { + sendMessage() + } label: { + Image(systemName: "arrow.up.circle.fill") + .foregroundStyle(inputText.isEmpty ? .gray : .accentColor) + } + .buttonStyle(.plain) + .disabled(inputText.isEmpty) + } + } + .padding() + } + .frame(width: 400, height: 500) + .onAppear { + isInputFocused = true + } + } + + private func sendMessage() { + let text = inputText + inputText = "" + viewModel.send(message: text) + } +} + +// MARK: - Panel Message Row + +private struct PanelMessageRow: View { + let message: AssistantMessage + + var body: some View { + HStack(alignment: .top, spacing: 6) { + switch message.role { + case .user: + Image(systemName: "person.circle.fill") + .foregroundStyle(.blue) + .font(.caption) + case .assistant: + Image(systemName: "sparkles") + .foregroundStyle(.purple) + .font(.caption) + case .system: + Image(systemName: "info.circle") + .foregroundStyle(.secondary) + .font(.caption) + } + + Text(message.content) + .font(.callout) + .textSelection(.enabled) + .foregroundStyle(message.role == .system ? .secondary : .primary) + + Spacer(minLength: 20) + } + } +} diff --git a/ArkavoCreator/ArkavoCreator/Assistant/AssistantPromptBuilder.swift b/ArkavoCreator/ArkavoCreator/Assistant/AssistantPromptBuilder.swift new file mode 100644 index 00000000..cd91226b --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Assistant/AssistantPromptBuilder.swift @@ -0,0 +1,88 @@ +import Foundation + +/// Composes system prompts for the content creation assistant +enum AssistantPromptBuilder { + /// Build a system prompt incorporating platform context + static func buildSystemPrompt(for context: any PlatformContext) -> String { + """ + You are a content creation assistant for a social media creator. \ + You help draft, rewrite, and adapt content across platforms. \ + Be concise, creative, and match the tone appropriate for each platform. + + \(context.systemPromptFragment) + + Guidelines: + - Be direct and helpful. Skip preamble. + - When drafting, provide ready-to-use content. + - When rewriting, preserve the core message while improving clarity and engagement. + - Respect character limits strictly when specified. + - Suggest hashtags, keywords, or formatting only when relevant to the platform. + """ + } + + /// Build a prompt with conversation history + static func buildPrompt( + userMessage: String, + context: any PlatformContext, + conversationHistory: [(role: String, content: String)] = [] + ) -> String { + var prompt = "" + + if !conversationHistory.isEmpty { + for entry in conversationHistory.suffix(10) { + prompt += "\(entry.role): \(entry.content)\n" + } + prompt += "\n" + } + + prompt += userMessage + return prompt + } + + /// Build a prompt for a specific action + static func buildActionPrompt( + action: AssistantAction, + inputText: String?, + context: any PlatformContext + ) -> String { + let platformInfo = context.characterLimit.map { "Maximum \($0) characters. " } ?? "" + + switch action { + case .draftPost: + if let input = inputText, !input.isEmpty { + return "Draft a \(context.platformName) post about: \(input). \(platformInfo)" + } + return "Draft an engaging \(context.platformName) post. \(platformInfo)" + + case .rewrite: + guard let input = inputText, !input.isEmpty else { + return "Please provide text to rewrite." + } + return "Rewrite this for \(context.platformName): \(input). \(platformInfo)" + + case .adjustTone: + guard let input = inputText, !input.isEmpty else { + return "Please provide text to adjust." + } + return "Adjust the tone of this text to be more engaging for \(context.platformName): \(input). \(platformInfo)" + + case .adaptCrossPlatform: + guard let input = inputText, !input.isEmpty else { + return "Please provide content to adapt." + } + return "Adapt this content for \(context.platformName): \(input). \(platformInfo)" + + case .generateTitle: + if let input = inputText, !input.isEmpty { + return "Generate a compelling title for \(context.platformName) about: \(input)" + } + return "Generate a compelling title for \(context.platformName) content." + + case .generateDescription: + if let input = inputText, !input.isEmpty { + return "Generate a description for \(context.platformName) about: \(input)" + } + return "Generate a description for \(context.platformName) content." + } + } +} diff --git a/ArkavoCreator/ArkavoCreator/Assistant/AssistantViewModel.swift b/ArkavoCreator/ArkavoCreator/Assistant/AssistantViewModel.swift new file mode 100644 index 00000000..0d2577e5 --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Assistant/AssistantViewModel.swift @@ -0,0 +1,148 @@ +import Foundation +import MuseCore +import Observation + +/// Message in the assistant conversation +struct AssistantMessage: Identifiable, Sendable { + let id = UUID() + let role: Role + let content: String + let timestamp = Date() + + enum Role: Sendable { + case user + case assistant + case system + } +} + +/// View model for the AI assistant +@Observable +@MainActor +final class AssistantViewModel { + private(set) var messages: [AssistantMessage] = [] + private(set) var streamingText = "" + private(set) var isGenerating = false + + let modelManager: ModelManager + + private var platformContext: any PlatformContext = GenericContext() + private var generationTask: Task? + + init(modelManager: ModelManager) { + self.modelManager = modelManager + } + + /// Update the platform context when navigation changes + func updateContext(_ section: NavigationSection) { + platformContext = section.platformContext + + // Auto-unload large models when entering Studio + if section == .studio, + modelManager.selectedModel.estimatedMemoryMB > 2000, + modelManager.isReady + { + Task { + await modelManager.unloadModel(reason: "Unloaded for recording performance") + } + } + } + + /// Send a user message and generate a response + func send(message: String) { + guard !message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } + + messages.append(AssistantMessage(role: .user, content: message)) + + guard modelManager.isReady else { + messages.append(AssistantMessage( + role: .system, + content: "Model not loaded. Tap the model indicator to load." + )) + return + } + + isGenerating = true + streamingText = "" + + generationTask = Task { + let systemPrompt = AssistantPromptBuilder.buildSystemPrompt(for: platformContext) + let history = messages.suffix(10).compactMap { msg -> (role: String, content: String)? in + switch msg.role { + case .user: ("User", msg.content) + case .assistant: ("Assistant", msg.content) + case .system: nil + } + } + let prompt = AssistantPromptBuilder.buildPrompt( + userMessage: message, + context: platformContext, + conversationHistory: history + ) + + let stream = modelManager.streamingProvider.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: 512 + ) + + do { + for try await token in stream { + streamingText += token + } + // Finalize + let finalText = streamingText + messages.append(AssistantMessage(role: .assistant, content: finalText)) + } catch is CancellationError { + if !streamingText.isEmpty { + messages.append(AssistantMessage(role: .assistant, content: streamingText)) + } + } catch { + if !streamingText.isEmpty { + messages.append(AssistantMessage(role: .assistant, content: streamingText)) + } else { + messages.append(AssistantMessage(role: .system, content: "Error: \(error.localizedDescription)")) + } + } + + streamingText = "" + isGenerating = false + } + } + + /// Perform a quick action + func performAction(_ action: AssistantAction, inputText: String? = nil) { + let prompt = AssistantPromptBuilder.buildActionPrompt( + action: action, + inputText: inputText, + context: platformContext + ) + send(message: prompt) + } + + /// Regenerate the last assistant response + func regenerate() { + // Find the last user message + guard let lastUserMessage = messages.last(where: { $0.role == .user }) else { return } + + // Remove last assistant message if present + if let lastIndex = messages.indices.last, messages[lastIndex].role == .assistant { + messages.removeLast() + } + + send(message: lastUserMessage.content) + } + + /// Stop generation + func stopGeneration() { + generationTask?.cancel() + generationTask = nil + } + + /// Clear conversation + func clearConversation() { + stopGeneration() + messages.removeAll() + streamingText = "" + } +} diff --git a/ArkavoCreator/ArkavoCreator/Assistant/PlatformContext.swift b/ArkavoCreator/ArkavoCreator/Assistant/PlatformContext.swift new file mode 100644 index 00000000..678b1be4 --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Assistant/PlatformContext.swift @@ -0,0 +1,135 @@ +import Foundation + +/// Describes platform-specific constraints and actions for the AI assistant +protocol PlatformContext { + var platformName: String { get } + var systemPromptFragment: String { get } + var characterLimit: Int? { get } + var suggestedActions: [AssistantAction] { get } +} + +/// Actions the assistant can perform based on the current platform +enum AssistantAction: String, CaseIterable, Sendable { + case draftPost = "Draft Post" + case rewrite = "Rewrite" + case adjustTone = "Adjust Tone" + case adaptCrossPlatform = "Adapt to Platform" + case generateTitle = "Generate Title" + case generateDescription = "Generate Description" +} + +// MARK: - Platform Contexts + +struct BlueskyContext: PlatformContext { + let platformName = "Bluesky" + let characterLimit: Int? = 300 + let suggestedActions: [AssistantAction] = [.draftPost, .rewrite, .adjustTone, .adaptCrossPlatform] + var systemPromptFragment: String { + """ + Currently helping with: Bluesky + Post constraints: Maximum 300 characters. Supports mentions (@handle) and links. + Style: Casual, engaging, concise. Hashtags are not commonly used on Bluesky. + """ + } +} + +struct YouTubeContext: PlatformContext { + let platformName = "YouTube" + let characterLimit: Int? = 5000 + let suggestedActions: [AssistantAction] = [.generateTitle, .generateDescription, .adjustTone, .adaptCrossPlatform] + var systemPromptFragment: String { + """ + Currently helping with: YouTube + Title: Max 100 characters, SEO-friendly, attention-grabbing. + Description: Up to 5000 characters. First 2-3 lines most important (shown before "Show more"). + Include relevant keywords, timestamps, links, and calls-to-action. + Tags: Relevant keywords for discoverability. + """ + } +} + +struct TwitchContext: PlatformContext { + let platformName = "Twitch" + let characterLimit: Int? = 140 + let suggestedActions: [AssistantAction] = [.generateTitle, .draftPost, .adjustTone] + var systemPromptFragment: String { + """ + Currently helping with: Twitch + Stream title: Maximum 140 characters. Should be engaging and descriptive. + Tags: Up to 10 tags for discoverability. + Style: Energetic, community-focused, often uses emotes and casual language. + """ + } +} + +struct RedditContext: PlatformContext { + let platformName = "Reddit" + let characterLimit: Int? = nil + let suggestedActions: [AssistantAction] = [.draftPost, .generateTitle, .rewrite, .adjustTone] + var systemPromptFragment: String { + """ + Currently helping with: Reddit + Title: Concise, descriptive, follows subreddit conventions. + Body: Supports Markdown. Length varies by subreddit norms. + Style: Authentic, community-aware. Avoid overly promotional language. + """ + } +} + +struct MicropubContext: PlatformContext { + let platformName = "Micro.blog" + let characterLimit: Int? = nil + let suggestedActions: [AssistantAction] = [.draftPost, .rewrite, .adjustTone, .generateTitle] + var systemPromptFragment: String { + """ + Currently helping with: Micro.blog / Micropub + Supports HTML and Markdown. Blog-style content. + Style: Thoughtful, personal voice. Can be long-form or microblog (< 280 chars for timeline). + """ + } +} + +struct LibraryContext: PlatformContext { + let platformName = "Library" + let characterLimit: Int? = nil + let suggestedActions: [AssistantAction] = [.generateTitle, .generateDescription] + var systemPromptFragment: String { + """ + Currently helping with: Recording Library + Generate titles and descriptions for recorded videos. + Style: Clear, descriptive, professional. + """ + } +} + +struct GenericContext: PlatformContext { + let platformName = "General" + let characterLimit: Int? = nil + let suggestedActions: [AssistantAction] = [.draftPost, .rewrite, .adjustTone, .adaptCrossPlatform] + var systemPromptFragment: String { + """ + Currently in general mode. Help with any content creation task. + Available platforms: Bluesky, YouTube, Twitch, Reddit, Micro.blog. + """ + } +} + +// MARK: - Navigation Section Extension + +extension NavigationSection { + /// Get the appropriate platform context for this section + var platformContext: any PlatformContext { + switch self { + case .dashboard: GenericContext() + case .profile: GenericContext() + case .studio: GenericContext() + case .library: LibraryContext() + case .workflow: GenericContext() + case .assistant: GenericContext() + case .patrons: GenericContext() + case .protection: GenericContext() + case .social: GenericContext() + case .settings: GenericContext() + } + } +} diff --git a/ArkavoCreator/ArkavoCreator/ContentView.swift b/ArkavoCreator/ArkavoCreator/ContentView.swift index 671d5a18..db5e2e6a 100644 --- a/ArkavoCreator/ArkavoCreator/ContentView.swift +++ b/ArkavoCreator/ArkavoCreator/ContentView.swift @@ -13,10 +13,12 @@ struct ContentView: View { @StateObject var blueskyClient: BlueskyClient @StateObject var youtubeClient: YouTubeClient @ObservedObject var agentService: CreatorAgentService + var assistantViewModel: AssistantViewModel @StateObject private var twitchClient = TwitchAuthClient( clientId: Secrets.twitchClientId, clientSecret: Secrets.twitchClientSecret ) + @State private var showAssistantPanel = false var body: some View { NavigationSplitView { @@ -36,14 +38,32 @@ struct ContentView: View { blueskyClient: blueskyClient, youtubeClient: youtubeClient, twitchClient: twitchClient, - agentService: agentService + agentService: agentService, + assistantViewModel: assistantViewModel ) .navigationTitle(selectedSection.rawValue) .navigationSubtitle(selectedSection.subtitle) + .toolbar { + if FeatureFlags.localAssistant { + ToolbarItem(placement: .primaryAction) { + Button { + showAssistantPanel.toggle() + } label: { + Image(systemName: "wand.and.stars") + } + .keyboardShortcut("a", modifiers: [.command, .shift]) + .help("AI Assistant (⌘⇧A)") + } + } + } + .sheet(isPresented: $showAssistantPanel) { + AssistantPanelView(viewModel: assistantViewModel) + } } .environmentObject(appState) .onChange(of: selectedSection) { _, newValue in UserDefaults.standard.saveSelectedTab(newValue) + assistantViewModel.updateContext(newValue) } } } @@ -68,7 +88,7 @@ enum NavigationSection: String, CaseIterable, Codable { case .workflow: return FeatureFlags.workflow case .protection: return FeatureFlags.contentProtection case .social: return FeatureFlags.social - case .assistant: return FeatureFlags.aiAgent + case .assistant: return FeatureFlags.aiAgent || FeatureFlags.localAssistant case .patrons: return FeatureFlags.patreon default: return true } @@ -137,6 +157,7 @@ struct SectionContainer: View { @ObservedObject var youtubeClient: YouTubeClient @ObservedObject var twitchClient: TwitchAuthClient @ObservedObject var agentService: CreatorAgentService + var assistantViewModel: AssistantViewModel @StateObject private var webViewPresenter = WebViewPresenter() @Namespace private var animation @@ -851,9 +872,15 @@ struct SectionContainer: View { .transition(.moveAndFade()) .id("content") case .assistant: - AssistantSectionView(agentService: agentService) - .transition(.moveAndFade()) - .id("assistant") + if FeatureFlags.localAssistant { + AssistantChatView(viewModel: assistantViewModel) + .transition(.moveAndFade()) + .id("local-assistant") + } else { + AssistantSectionView(agentService: agentService) + .transition(.moveAndFade()) + .id("assistant") + } case .settings: SettingsContent(agentService: agentService) .transition(.moveAndFade()) diff --git a/ArkavoCreator/ArkavoCreator/FeatureFlags.swift b/ArkavoCreator/ArkavoCreator/FeatureFlags.swift index 0bef64d2..de0f5811 100644 --- a/ArkavoCreator/ArkavoCreator/FeatureFlags.swift +++ b/ArkavoCreator/ArkavoCreator/FeatureFlags.swift @@ -21,4 +21,6 @@ enum FeatureFlags { static let workflow = false /// Marketing/social section static let social = false + /// Local on-device AI assistant (MLX) + static let localAssistant = true } diff --git a/MuseCore/Package.resolved b/MuseCore/Package.resolved index 0e14dcb7..6c472929 100644 --- a/MuseCore/Package.resolved +++ b/MuseCore/Package.resolved @@ -1,6 +1,69 @@ { - "originHash" : "bada40e31b8394c8d2c6a989fed977cf2cf6ff3cc6546dd934cae3a5619e6fcc", + "originHash" : "47118e32930d6dfe6cf25b3c75b765c32a01978ddd3e3b9132af7f2d27fd500d", "pins" : [ + { + "identity" : "gzipswift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/1024jp/GzipSwift", + "state" : { + "revision" : "731037f6cc2be2ec01562f6597c1d0aa3fe6fd05", + "version" : "6.0.1" + } + }, + { + "identity" : "mlx-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift", + "state" : { + "revision" : "072b684acaae80b6a463abab3a103732f33774bf", + "version" : "0.29.1" + } + }, + { + "identity" : "mlx-swift-examples", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift-examples", + "state" : { + "revision" : "9bff95ca5f0b9e8c021acc4d71a2bbe4a7441631", + "version" : "2.29.1" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "8d9834a6189db730f6264db7556a7ffb751e99ee", + "version" : "1.4.0" + } + }, + { + "identity" : "swift-jinja", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-jinja.git", + "state" : { + "revision" : "f731f03bf746481d4fda07f817c3774390c4d5b9", + "version" : "2.3.2" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-transformers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-transformers", + "state" : { + "revision" : "a2e184dddb4757bc943e77fbe99ac6786c53f0b2", + "version" : "1.0.0" + } + }, { "identity" : "vrmmetalkit", "kind" : "remoteSourceControl", diff --git a/MuseCore/Package.swift b/MuseCore/Package.swift index 6c756423..784a9a53 100644 --- a/MuseCore/Package.swift +++ b/MuseCore/Package.swift @@ -14,12 +14,17 @@ let package = Package( ) ], dependencies: [ - .package(url: "https://github.com/arkavo-org/VRMMetalKit", exact: "0.9.2") + .package(url: "https://github.com/arkavo-org/VRMMetalKit", exact: "0.9.2"), + .package(url: "https://github.com/ml-explore/mlx-swift-examples", from: "2.29.1"), ], targets: [ .target( name: "MuseCore", - dependencies: ["VRMMetalKit"], + dependencies: [ + "VRMMetalKit", + .product(name: "MLXLLM", package: "mlx-swift-examples"), + .product(name: "MLXLMCommon", package: "mlx-swift-examples"), + ], swiftSettings: [ .enableExperimentalFeature("StrictConcurrency") ] diff --git a/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift new file mode 100644 index 00000000..6a236753 --- /dev/null +++ b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift @@ -0,0 +1,125 @@ +import Foundation +import MLX +import MLXLMCommon +import MLXLLM +import Synchronization + +/// MLX-based streaming LLM provider for on-device inference. +/// Wraps mlx-swift-examples v2 model loading and generation. +public final class MLXBackend: StreamingLLMProvider, @unchecked Sendable { + private let state = Mutex(BackendState()) + + public let providerName = "MLX Local" + + public init() {} + + public var isAvailable: Bool { + get async { + state.withLock { $0.modelContainer != nil } + } + } + + /// Load a model by HuggingFace ID + public func loadModel(_ huggingFaceID: String) async throws { + let container = try await MLXLMCommon.loadModelContainer( + id: huggingFaceID + ) { progress in + debugPrint("Loading \(huggingFaceID): \(Int(progress.fractionCompleted * 100))%") + } + + // Set memory limit to 75% of system RAM for safety + let systemMemoryGB = ProcessInfo.processInfo.physicalMemory / (1024 * 1024 * 1024) + let limitBytes = Int(Double(systemMemoryGB) * 0.75) * 1024 * 1024 * 1024 + MLX.GPU.set(memoryLimit: limitBytes) + + state.withLock { $0.modelContainer = container } + } + + /// Unload the current model to free GPU memory + public func unloadModel() { + state.withLock { $0.modelContainer = nil } + MLX.GPU.clearCache() + } + + public func generate( + prompt: String, + systemPrompt: String, + maxTokens: Int + ) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + let task = Task { [weak self] in + guard let self else { + continuation.finish(throwing: StreamingLLMError.modelNotLoaded) + return + } + + guard let container = self.state.withLock({ $0.modelContainer }) else { + continuation.finish(throwing: StreamingLLMError.modelNotLoaded) + return + } + + do { + let userInput = UserInput(chat: [ + .system(systemPrompt), + .user(prompt), + ]) + + let parameters = GenerateParameters( + maxTokens: maxTokens, + temperature: 0.7, + topP: 0.9, + repetitionPenalty: 1.1 + ) + + try await container.perform { context in + let lmInput = try await context.processor.prepare(input: userInput) + let stream = try MLXLMCommon.generate( + input: lmInput, + parameters: parameters, + context: context + ) + + for await generation in stream { + if Task.isCancelled { break } + if let chunk = generation.chunk { + continuation.yield(chunk) + } + } + } + + continuation.finish() + } catch { + if Task.isCancelled { + continuation.finish(throwing: StreamingLLMError.generationCancelled) + } else { + continuation.finish(throwing: error) + } + } + } + + state.withLock { $0.generationTask = task } + + continuation.onTermination = { @Sendable _ in + task.cancel() + } + } + } + + public func cancelGeneration() async { + let task = state.withLock { s -> Task? in + let t = s.generationTask + s.generationTask = nil + return t + } + task?.cancel() + } +} + +// MARK: - Internal State + +private struct BackendState: ~Copyable { + var modelContainer: ModelContainer? + var generationTask: Task? + + init() {} +} diff --git a/MuseCore/Sources/MuseCore/LLM/ModelManager.swift b/MuseCore/Sources/MuseCore/LLM/ModelManager.swift new file mode 100644 index 00000000..27af3704 --- /dev/null +++ b/MuseCore/Sources/MuseCore/LLM/ModelManager.swift @@ -0,0 +1,94 @@ +import Foundation +import Observation + +/// State of model lifecycle +public enum ModelState: Equatable, Sendable { + case idle + case downloading(progress: Double) + case loading + case ready + case error(String) + case unloaded(reason: String) +} + +/// Manages MLX model lifecycle: download, load, unload, and memory budget. +@Observable +@MainActor +public final class ModelManager { + public private(set) var state: ModelState = .idle + public private(set) var selectedModel: ModelInfo = ModelRegistry.defaultModel + public private(set) var availableModels: [ModelInfo] = [] + + private let backend: MLXBackend + + /// The backend used by this manager — pass to AssistantViewModel + public var streamingProvider: MLXBackend { backend } + + public init() { + backend = MLXBackend() + refreshAvailableModels() + } + + /// Refresh which models are available based on system memory + public func refreshAvailableModels() { + let systemMemoryMB = Int(ProcessInfo.processInfo.physicalMemory / (1024 * 1024)) + // Use 50% of system memory as budget for model loading + let budgetMB = systemMemoryMB / 2 + availableModels = ModelRegistry.availableModels(memoryBudgetMB: budgetMB) + } + + /// Select and load a model + public func selectModel(_ model: ModelInfo) async { + guard model != selectedModel || state != .ready else { return } + + selectedModel = model + + // Unload current model first + if state == .ready { + await unloadModel() + } + + await loadSelectedModel() + } + + /// Load the currently selected model + public func loadSelectedModel() async { + guard state != .loading else { return } + + state = .loading + + do { + try await backend.loadModel(selectedModel.huggingFaceID) + state = .ready + } catch { + state = .error(error.localizedDescription) + } + } + + /// Unload the model to free GPU memory + public func unloadModel() async { + backend.unloadModel() + state = .idle + } + + /// Unload with a reason (e.g., entering Studio) + public func unloadModel(reason: String) async { + backend.unloadModel() + state = .unloaded(reason: reason) + } + + /// Whether the model is ready for generation + public var isReady: Bool { + state == .ready + } + + /// System memory in GB + public var systemMemoryGB: Int { + Int(ProcessInfo.processInfo.physicalMemory / (1024 * 1024 * 1024)) + } + + /// Whether the selected model is cached locally + public var isSelectedModelCached: Bool { + ModelRegistry.isModelCached(selectedModel) + } +} diff --git a/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift b/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift new file mode 100644 index 00000000..4065b5a3 --- /dev/null +++ b/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift @@ -0,0 +1,81 @@ +import Foundation + +/// Catalog of supported MLX models with metadata +public struct ModelInfo: Sendable, Identifiable, Hashable { + public let id: String + public let displayName: String + public let huggingFaceID: String + public let estimatedMemoryMB: Int + public let parameterCount: String + public let quantization: String + + public init( + id: String, + displayName: String, + huggingFaceID: String, + estimatedMemoryMB: Int, + parameterCount: String, + quantization: String + ) { + self.id = id + self.displayName = displayName + self.huggingFaceID = huggingFaceID + self.estimatedMemoryMB = estimatedMemoryMB + self.parameterCount = parameterCount + self.quantization = quantization + } +} + +/// Registry of available MLX models +public enum ModelRegistry { + /// All supported models, ordered by size + public static let models: [ModelInfo] = [ + ModelInfo( + id: "gemma-3-270m", + displayName: "Gemma 3 270M", + huggingFaceID: "mlx-community/gemma-3-270m-it-bf16", + estimatedMemoryMB: 906, + parameterCount: "270M", + quantization: "bf16" + ), + ModelInfo( + id: "qwen3.5-0.8b", + displayName: "Qwen 3.5 0.8B", + huggingFaceID: "mlx-community/Qwen3.5-0.8B", + estimatedMemoryMB: 1600, + parameterCount: "0.8B", + quantization: "bf16" + ), + ModelInfo( + id: "qwen3.5-9b", + displayName: "Qwen 3.5 9B", + huggingFaceID: "mlx-community/Qwen3.5-9B", + estimatedMemoryMB: 18000, + parameterCount: "9B", + quantization: "bf16" + ), + ] + + /// The default model (smallest, already cached) + public static let defaultModel = models[0] + + /// Find a model by its ID + public static func model(forID id: String) -> ModelInfo? { + models.first { $0.id == id } + } + + /// Models that fit within the given memory budget (in MB) + public static func availableModels(memoryBudgetMB: Int) -> [ModelInfo] { + models.filter { $0.estimatedMemoryMB <= memoryBudgetMB } + } + + /// Check if a model's files exist in the HuggingFace cache + public static func isModelCached(_ model: ModelInfo) -> Bool { + let cacheDir = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent(".cache/huggingface/hub") + let repoDir = cacheDir.appendingPathComponent( + "models--\(model.huggingFaceID.replacingOccurrences(of: "/", with: "--"))" + ) + return FileManager.default.fileExists(atPath: repoDir.path) + } +} diff --git a/MuseCore/Sources/MuseCore/LLM/StreamingLLMProvider.swift b/MuseCore/Sources/MuseCore/LLM/StreamingLLMProvider.swift new file mode 100644 index 00000000..a514c70e --- /dev/null +++ b/MuseCore/Sources/MuseCore/LLM/StreamingLLMProvider.swift @@ -0,0 +1,47 @@ +import Foundation + +/// Protocol for streaming LLM providers that return raw token streams. +/// Unlike `LLMResponseProvider` which returns structured `ConstrainedResponse`, +/// this protocol is designed for content creation tasks needing raw text output. +public protocol StreamingLLMProvider: Sendable { + /// Whether the provider is ready to generate + var isAvailable: Bool { get async } + + /// Human-readable provider name + var providerName: String { get } + + /// Generate a streaming response + /// - Parameters: + /// - prompt: The user prompt + /// - systemPrompt: System instructions for the model + /// - maxTokens: Maximum tokens to generate + /// - Returns: An async stream of token strings + func generate(prompt: String, systemPrompt: String, maxTokens: Int) -> AsyncThrowingStream + + /// Cancel any in-progress generation + func cancelGeneration() async +} + +/// Errors specific to streaming LLM operations +public enum StreamingLLMError: Error, LocalizedError { + case modelNotLoaded + case generationCancelled + case modelLoadFailed(String) + case insufficientMemory(required: Int, available: Int) + case downloadFailed(String) + + public var errorDescription: String? { + switch self { + case .modelNotLoaded: + "No model is currently loaded" + case .generationCancelled: + "Generation was cancelled" + case .modelLoadFailed(let reason): + "Failed to load model: \(reason)" + case .insufficientMemory(let required, let available): + "Insufficient memory: need \(required)MB, have \(available)MB" + case .downloadFailed(let reason): + "Download failed: \(reason)" + } + } +} From b43dac2b25862d9eb4833e888dec5455b2515b28 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Wed, 18 Mar 2026 20:58:01 -0400 Subject: [PATCH 02/14] Replace chat UI with Muse three-role system: Producer, Publicist, Sidekick MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transform the text chatbot into a role-based architecture where Muse fills three distinct roles for creators. Producer monitors streams via a private Studio overlay panel. Publicist generates platform-native content across six connected platforms. Sidekick (Phase 1.1b) will be the on-camera avatar. - Add AvatarRole enum and RolePromptProvider with per-role/locale prompts - Add MLXResponseProvider adapter (MLXBackend → LLMResponseProvider) - Add Producer panel (slide-in overlay in Studio, Cmd+P toggle) - Add Publicist view (content workspace replacing chat UI) - Add role-aware ConversationManager with context injection - Fix model auto-load on restart (sandbox-aware cache detection) - Fix concurrent model load race condition (generation counter) - Fix token leaking into output (stop sequence filtering) - Remove AssistantChatView, AssistantPanelView, AssistantViewModel - Remove StreamingLLMProvider protocol (inlined into MLXBackend) - Rename AssistantAction → PublicistAction Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ArkavoCreator/ArkavoCreatorApp.swift | 8 +- .../Assistant/AssistantChatView.swift | 273 --------------- .../Assistant/AssistantPanelView.swift | 157 --------- .../Assistant/AssistantViewModel.swift | 148 -------- .../Assistant/PlatformContext.swift | 26 +- ArkavoCreator/ArkavoCreator/ContentView.swift | 37 +- .../ArkavoCreator/FeatureFlags.swift | 2 +- .../Producer/ProducerPanelView.swift | 226 ++++++++++++ .../Producer/ProducerViewModel.swift | 132 +++++++ .../Producer/StreamStateContext.swift | 61 ++++ .../PublicistPromptBuilder.swift} | 8 +- .../Publicist/PublicistView.swift | 326 ++++++++++++++++++ .../Publicist/PublicistViewModel.swift | 148 ++++++++ ArkavoCreator/ArkavoCreator/RecordView.swift | 33 ++ .../Streaming/ChatPanelViewModel.swift | 28 ++ .../Streaming/StreamChatReactor.swift | 12 + .../ProducerUITests.swift | 85 +++++ .../PublicistUITests.swift | 70 ++++ .../Sources/MuseCore/LLM/AvatarRole.swift | 162 +++++++++ .../MuseCore/LLM/ConversationManager.swift | 35 +- .../Sources/MuseCore/LLM/MLXBackend.swift | 60 +++- .../MuseCore/LLM/MLXResponseProvider.swift | 100 ++++++ .../Sources/MuseCore/LLM/ModelManager.swift | 34 +- .../Sources/MuseCore/LLM/ModelRegistry.swift | 17 +- .../MuseCore/LLM/StreamingLLMProvider.swift | 47 --- 25 files changed, 1536 insertions(+), 699 deletions(-) delete mode 100644 ArkavoCreator/ArkavoCreator/Assistant/AssistantChatView.swift delete mode 100644 ArkavoCreator/ArkavoCreator/Assistant/AssistantPanelView.swift delete mode 100644 ArkavoCreator/ArkavoCreator/Assistant/AssistantViewModel.swift create mode 100644 ArkavoCreator/ArkavoCreator/Producer/ProducerPanelView.swift create mode 100644 ArkavoCreator/ArkavoCreator/Producer/ProducerViewModel.swift create mode 100644 ArkavoCreator/ArkavoCreator/Producer/StreamStateContext.swift rename ArkavoCreator/ArkavoCreator/{Assistant/AssistantPromptBuilder.swift => Publicist/PublicistPromptBuilder.swift} (92%) create mode 100644 ArkavoCreator/ArkavoCreator/Publicist/PublicistView.swift create mode 100644 ArkavoCreator/ArkavoCreator/Publicist/PublicistViewModel.swift create mode 100644 ArkavoCreator/ArkavoCreatorUITests/ProducerUITests.swift create mode 100644 ArkavoCreator/ArkavoCreatorUITests/PublicistUITests.swift create mode 100644 MuseCore/Sources/MuseCore/LLM/AvatarRole.swift create mode 100644 MuseCore/Sources/MuseCore/LLM/MLXResponseProvider.swift delete mode 100644 MuseCore/Sources/MuseCore/LLM/StreamingLLMProvider.swift diff --git a/ArkavoCreator/ArkavoCreator/ArkavoCreatorApp.swift b/ArkavoCreator/ArkavoCreator/ArkavoCreatorApp.swift index dd109d2f..b7c21001 100644 --- a/ArkavoCreator/ArkavoCreator/ArkavoCreatorApp.swift +++ b/ArkavoCreator/ArkavoCreator/ArkavoCreatorApp.swift @@ -20,7 +20,6 @@ struct ArkavoCreatorApp: App { @StateObject private var agentService = CreatorAgentService() @State private var modelManager = ModelManager() - @State private var assistantViewModel: AssistantViewModel? let patreonClient = PatreonClient(clientId: Secrets.patreonClientId, clientSecret: Secrets.patreonClientSecret) let redditClient = RedditClient(clientId: Secrets.redditClientId) @@ -58,14 +57,9 @@ struct ArkavoCreatorApp: App { blueskyClient: blueskyClient, youtubeClient: youtubeClient, agentService: agentService, - assistantViewModel: assistantViewModel ?? AssistantViewModel(modelManager: modelManager) + modelManager: modelManager ) .onAppear { - // Initialize assistant view model - if assistantViewModel == nil { - assistantViewModel = AssistantViewModel(modelManager: modelManager) - } - // Load stored tokens redditClient.loadStoredTokens() micropubClient.loadStoredTokens() diff --git a/ArkavoCreator/ArkavoCreator/Assistant/AssistantChatView.swift b/ArkavoCreator/ArkavoCreator/Assistant/AssistantChatView.swift deleted file mode 100644 index 19c81bea..00000000 --- a/ArkavoCreator/ArkavoCreator/Assistant/AssistantChatView.swift +++ /dev/null @@ -1,273 +0,0 @@ -import MuseCore -import SwiftUI - -/// Full AI assistant chat view for the Assistant section -struct AssistantChatView: View { - @Bindable var viewModel: AssistantViewModel - - @State private var inputText = "" - @FocusState private var isInputFocused: Bool - - var body: some View { - VStack(spacing: 0) { - // Model status bar - modelStatusBar - - // Platform context indicator - platformContextBar - - Divider() - - // Messages - ScrollViewReader { proxy in - ScrollView { - LazyVStack(alignment: .leading, spacing: 12) { - ForEach(viewModel.messages) { message in - MessageBubble(message: message) - .id(message.id) - } - - // Streaming indicator - if viewModel.isGenerating, !viewModel.streamingText.isEmpty { - MessageBubble( - message: AssistantMessage(role: .assistant, content: viewModel.streamingText) - ) - .id("streaming") - } - } - .padding() - } - .onChange(of: viewModel.messages.count) { - withAnimation { - if let lastID = viewModel.messages.last?.id { - proxy.scrollTo(lastID, anchor: .bottom) - } - } - } - .onChange(of: viewModel.streamingText) { - if viewModel.isGenerating { - proxy.scrollTo("streaming", anchor: .bottom) - } - } - } - - Divider() - - // Quick actions - quickActionsBar - - // Input bar - inputBar - } - } - - // MARK: - Components - - private var modelStatusBar: some View { - HStack(spacing: 8) { - Circle() - .fill(statusColor) - .frame(width: 8, height: 8) - - Text(statusText) - .font(.caption) - .foregroundStyle(.secondary) - - Spacer() - - // Model picker - Menu { - ForEach(viewModel.modelManager.availableModels) { model in - Button { - Task { await viewModel.modelManager.selectModel(model) } - } label: { - HStack { - Text(model.displayName) - if model == viewModel.modelManager.selectedModel { - Image(systemName: "checkmark") - } - } - } - } - } label: { - HStack(spacing: 4) { - Text(viewModel.modelManager.selectedModel.displayName) - .font(.caption) - Image(systemName: "chevron.down") - .font(.caption2) - } - .foregroundStyle(.secondary) - } - .menuStyle(.borderlessButton) - .fixedSize() - - // Load/unload button - if viewModel.modelManager.state == .idle || viewModel.modelManager.state == .error("") || isUnloaded { - Button("Load") { - Task { await viewModel.modelManager.loadSelectedModel() } - } - .buttonStyle(.bordered) - .controlSize(.small) - } - } - .padding(.horizontal) - .padding(.vertical, 8) - .background(.bar) - } - - private var isUnloaded: Bool { - if case .unloaded = viewModel.modelManager.state { return true } - return false - } - - private var statusColor: Color { - switch viewModel.modelManager.state { - case .ready: .green - case .loading, .downloading: .orange - case .error: .red - default: .gray - } - } - - private var statusText: String { - switch viewModel.modelManager.state { - case .idle: "Model not loaded" - case .downloading(let progress): "Downloading \(Int(progress * 100))%" - case .loading: "Loading model..." - case .ready: "Ready" - case .error(let msg): "Error: \(msg)" - case .unloaded(let reason): reason - } - } - - private var platformContextBar: some View { - let context = viewModel.modelManager.selectedModel.displayName - return HStack(spacing: 8) { - Image(systemName: "sparkles") - .font(.caption) - .foregroundStyle(.secondary) - Text("Local inference with \(context)") - .font(.caption) - .foregroundStyle(.secondary) - Spacer() - } - .padding(.horizontal) - .padding(.vertical, 4) - } - - private var quickActionsBar: some View { - ScrollView(.horizontal, showsIndicators: false) { - HStack(spacing: 8) { - ForEach(currentActions, id: \.self) { action in - Button(action.rawValue) { - viewModel.performAction(action) - } - .buttonStyle(.bordered) - .controlSize(.small) - .disabled(!viewModel.modelManager.isReady || viewModel.isGenerating) - } - } - .padding(.horizontal) - .padding(.vertical, 6) - } - } - - private var currentActions: [AssistantAction] { - // Get actions from the current navigation context - GenericContext().suggestedActions - } - - private var inputBar: some View { - HStack(spacing: 8) { - TextField("Ask the assistant...", text: $inputText, axis: .vertical) - .textFieldStyle(.plain) - .lineLimit(1...5) - .focused($isInputFocused) - .onSubmit { - sendMessage() - } - - if viewModel.isGenerating { - Button { - viewModel.stopGeneration() - } label: { - Image(systemName: "stop.circle.fill") - .foregroundStyle(.red) - } - .buttonStyle(.plain) - } else { - Button { - sendMessage() - } label: { - Image(systemName: "arrow.up.circle.fill") - .foregroundStyle(inputText.isEmpty ? .gray : .accentColor) - } - .buttonStyle(.plain) - .disabled(inputText.isEmpty) - } - } - .padding() - } - - private func sendMessage() { - let text = inputText - inputText = "" - viewModel.send(message: text) - } -} - -// MARK: - Message Bubble - -private struct MessageBubble: View { - let message: AssistantMessage - - var body: some View { - HStack(alignment: .top, spacing: 10) { - avatar - - VStack(alignment: .leading, spacing: 4) { - Text(message.content) - .textSelection(.enabled) - .font(message.role == .system ? .caption : .body) - .foregroundStyle(message.role == .system ? .secondary : .primary) - - Text(message.timestamp, style: .time) - .font(.caption2) - .foregroundStyle(.tertiary) - } - - Spacer(minLength: 40) - - if message.role == .assistant { - Button { - NSPasteboard.general.clearContents() - NSPasteboard.general.setString(message.content, forType: .string) - } label: { - Image(systemName: "doc.on.doc") - .font(.caption) - .foregroundStyle(.secondary) - } - .buttonStyle(.plain) - } - } - .padding(.horizontal, 4) - } - - @ViewBuilder - private var avatar: some View { - switch message.role { - case .user: - Image(systemName: "person.circle.fill") - .foregroundStyle(.blue) - .font(.title3) - case .assistant: - Image(systemName: "sparkles") - .foregroundStyle(.purple) - .font(.title3) - case .system: - Image(systemName: "info.circle") - .foregroundStyle(.secondary) - .font(.title3) - } - } -} diff --git a/ArkavoCreator/ArkavoCreator/Assistant/AssistantPanelView.swift b/ArkavoCreator/ArkavoCreator/Assistant/AssistantPanelView.swift deleted file mode 100644 index 6ee56841..00000000 --- a/ArkavoCreator/ArkavoCreator/Assistant/AssistantPanelView.swift +++ /dev/null @@ -1,157 +0,0 @@ -import MuseCore -import SwiftUI - -/// Compact floating panel for quick assistant access -struct AssistantPanelView: View { - @Bindable var viewModel: AssistantViewModel - - @State private var inputText = "" - @FocusState private var isInputFocused: Bool - @Environment(\.dismiss) private var dismiss - - var body: some View { - VStack(spacing: 0) { - // Header - HStack { - Label("AI Assistant", systemImage: "sparkles") - .font(.headline) - Spacer() - - // Model status - HStack(spacing: 4) { - Circle() - .fill(viewModel.modelManager.isReady ? .green : .gray) - .frame(width: 6, height: 6) - Text(viewModel.modelManager.selectedModel.displayName) - .font(.caption) - .foregroundStyle(.secondary) - } - - Button { - dismiss() - } label: { - Image(systemName: "xmark.circle.fill") - .foregroundStyle(.secondary) - } - .buttonStyle(.plain) - } - .padding() - - Divider() - - // Recent messages (last 5) - ScrollViewReader { proxy in - ScrollView { - LazyVStack(alignment: .leading, spacing: 8) { - ForEach(viewModel.messages.suffix(5)) { message in - PanelMessageRow(message: message) - .id(message.id) - } - - if viewModel.isGenerating, !viewModel.streamingText.isEmpty { - PanelMessageRow( - message: AssistantMessage(role: .assistant, content: viewModel.streamingText) - ) - .id("streaming") - } - } - .padding(.horizontal) - .padding(.vertical, 8) - } - .onChange(of: viewModel.messages.count) { - if let lastID = viewModel.messages.last?.id { - proxy.scrollTo(lastID, anchor: .bottom) - } - } - } - - Divider() - - // Quick actions - ScrollView(.horizontal, showsIndicators: false) { - HStack(spacing: 6) { - ForEach(AssistantAction.allCases.prefix(4), id: \.self) { action in - Button(action.rawValue) { - viewModel.performAction(action) - } - .buttonStyle(.bordered) - .controlSize(.mini) - .disabled(!viewModel.modelManager.isReady || viewModel.isGenerating) - } - } - .padding(.horizontal) - .padding(.vertical, 6) - } - - // Input - HStack(spacing: 8) { - TextField("Ask something...", text: $inputText) - .textFieldStyle(.plain) - .focused($isInputFocused) - .onSubmit { sendMessage() } - - if viewModel.isGenerating { - Button { - viewModel.stopGeneration() - } label: { - Image(systemName: "stop.circle.fill") - .foregroundStyle(.red) - } - .buttonStyle(.plain) - } else { - Button { - sendMessage() - } label: { - Image(systemName: "arrow.up.circle.fill") - .foregroundStyle(inputText.isEmpty ? .gray : .accentColor) - } - .buttonStyle(.plain) - .disabled(inputText.isEmpty) - } - } - .padding() - } - .frame(width: 400, height: 500) - .onAppear { - isInputFocused = true - } - } - - private func sendMessage() { - let text = inputText - inputText = "" - viewModel.send(message: text) - } -} - -// MARK: - Panel Message Row - -private struct PanelMessageRow: View { - let message: AssistantMessage - - var body: some View { - HStack(alignment: .top, spacing: 6) { - switch message.role { - case .user: - Image(systemName: "person.circle.fill") - .foregroundStyle(.blue) - .font(.caption) - case .assistant: - Image(systemName: "sparkles") - .foregroundStyle(.purple) - .font(.caption) - case .system: - Image(systemName: "info.circle") - .foregroundStyle(.secondary) - .font(.caption) - } - - Text(message.content) - .font(.callout) - .textSelection(.enabled) - .foregroundStyle(message.role == .system ? .secondary : .primary) - - Spacer(minLength: 20) - } - } -} diff --git a/ArkavoCreator/ArkavoCreator/Assistant/AssistantViewModel.swift b/ArkavoCreator/ArkavoCreator/Assistant/AssistantViewModel.swift deleted file mode 100644 index 0d2577e5..00000000 --- a/ArkavoCreator/ArkavoCreator/Assistant/AssistantViewModel.swift +++ /dev/null @@ -1,148 +0,0 @@ -import Foundation -import MuseCore -import Observation - -/// Message in the assistant conversation -struct AssistantMessage: Identifiable, Sendable { - let id = UUID() - let role: Role - let content: String - let timestamp = Date() - - enum Role: Sendable { - case user - case assistant - case system - } -} - -/// View model for the AI assistant -@Observable -@MainActor -final class AssistantViewModel { - private(set) var messages: [AssistantMessage] = [] - private(set) var streamingText = "" - private(set) var isGenerating = false - - let modelManager: ModelManager - - private var platformContext: any PlatformContext = GenericContext() - private var generationTask: Task? - - init(modelManager: ModelManager) { - self.modelManager = modelManager - } - - /// Update the platform context when navigation changes - func updateContext(_ section: NavigationSection) { - platformContext = section.platformContext - - // Auto-unload large models when entering Studio - if section == .studio, - modelManager.selectedModel.estimatedMemoryMB > 2000, - modelManager.isReady - { - Task { - await modelManager.unloadModel(reason: "Unloaded for recording performance") - } - } - } - - /// Send a user message and generate a response - func send(message: String) { - guard !message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } - - messages.append(AssistantMessage(role: .user, content: message)) - - guard modelManager.isReady else { - messages.append(AssistantMessage( - role: .system, - content: "Model not loaded. Tap the model indicator to load." - )) - return - } - - isGenerating = true - streamingText = "" - - generationTask = Task { - let systemPrompt = AssistantPromptBuilder.buildSystemPrompt(for: platformContext) - let history = messages.suffix(10).compactMap { msg -> (role: String, content: String)? in - switch msg.role { - case .user: ("User", msg.content) - case .assistant: ("Assistant", msg.content) - case .system: nil - } - } - let prompt = AssistantPromptBuilder.buildPrompt( - userMessage: message, - context: platformContext, - conversationHistory: history - ) - - let stream = modelManager.streamingProvider.generate( - prompt: prompt, - systemPrompt: systemPrompt, - maxTokens: 512 - ) - - do { - for try await token in stream { - streamingText += token - } - // Finalize - let finalText = streamingText - messages.append(AssistantMessage(role: .assistant, content: finalText)) - } catch is CancellationError { - if !streamingText.isEmpty { - messages.append(AssistantMessage(role: .assistant, content: streamingText)) - } - } catch { - if !streamingText.isEmpty { - messages.append(AssistantMessage(role: .assistant, content: streamingText)) - } else { - messages.append(AssistantMessage(role: .system, content: "Error: \(error.localizedDescription)")) - } - } - - streamingText = "" - isGenerating = false - } - } - - /// Perform a quick action - func performAction(_ action: AssistantAction, inputText: String? = nil) { - let prompt = AssistantPromptBuilder.buildActionPrompt( - action: action, - inputText: inputText, - context: platformContext - ) - send(message: prompt) - } - - /// Regenerate the last assistant response - func regenerate() { - // Find the last user message - guard let lastUserMessage = messages.last(where: { $0.role == .user }) else { return } - - // Remove last assistant message if present - if let lastIndex = messages.indices.last, messages[lastIndex].role == .assistant { - messages.removeLast() - } - - send(message: lastUserMessage.content) - } - - /// Stop generation - func stopGeneration() { - generationTask?.cancel() - generationTask = nil - } - - /// Clear conversation - func clearConversation() { - stopGeneration() - messages.removeAll() - streamingText = "" - } -} diff --git a/ArkavoCreator/ArkavoCreator/Assistant/PlatformContext.swift b/ArkavoCreator/ArkavoCreator/Assistant/PlatformContext.swift index 678b1be4..3d47342c 100644 --- a/ArkavoCreator/ArkavoCreator/Assistant/PlatformContext.swift +++ b/ArkavoCreator/ArkavoCreator/Assistant/PlatformContext.swift @@ -1,15 +1,15 @@ import Foundation -/// Describes platform-specific constraints and actions for the AI assistant +/// Describes platform-specific constraints and actions for the Publicist protocol PlatformContext { var platformName: String { get } var systemPromptFragment: String { get } var characterLimit: Int? { get } - var suggestedActions: [AssistantAction] { get } + var suggestedActions: [PublicistAction] { get } } -/// Actions the assistant can perform based on the current platform -enum AssistantAction: String, CaseIterable, Sendable { +/// Actions the Publicist can perform based on the current platform +enum PublicistAction: String, CaseIterable, Sendable { case draftPost = "Draft Post" case rewrite = "Rewrite" case adjustTone = "Adjust Tone" @@ -23,7 +23,7 @@ enum AssistantAction: String, CaseIterable, Sendable { struct BlueskyContext: PlatformContext { let platformName = "Bluesky" let characterLimit: Int? = 300 - let suggestedActions: [AssistantAction] = [.draftPost, .rewrite, .adjustTone, .adaptCrossPlatform] + let suggestedActions: [PublicistAction] = [.draftPost, .rewrite, .adjustTone, .adaptCrossPlatform] var systemPromptFragment: String { """ Currently helping with: Bluesky @@ -36,7 +36,7 @@ struct BlueskyContext: PlatformContext { struct YouTubeContext: PlatformContext { let platformName = "YouTube" let characterLimit: Int? = 5000 - let suggestedActions: [AssistantAction] = [.generateTitle, .generateDescription, .adjustTone, .adaptCrossPlatform] + let suggestedActions: [PublicistAction] = [.generateTitle, .generateDescription, .adjustTone, .adaptCrossPlatform] var systemPromptFragment: String { """ Currently helping with: YouTube @@ -51,7 +51,7 @@ struct YouTubeContext: PlatformContext { struct TwitchContext: PlatformContext { let platformName = "Twitch" let characterLimit: Int? = 140 - let suggestedActions: [AssistantAction] = [.generateTitle, .draftPost, .adjustTone] + let suggestedActions: [PublicistAction] = [.generateTitle, .draftPost, .adjustTone] var systemPromptFragment: String { """ Currently helping with: Twitch @@ -65,7 +65,7 @@ struct TwitchContext: PlatformContext { struct RedditContext: PlatformContext { let platformName = "Reddit" let characterLimit: Int? = nil - let suggestedActions: [AssistantAction] = [.draftPost, .generateTitle, .rewrite, .adjustTone] + let suggestedActions: [PublicistAction] = [.draftPost, .generateTitle, .rewrite, .adjustTone] var systemPromptFragment: String { """ Currently helping with: Reddit @@ -79,7 +79,7 @@ struct RedditContext: PlatformContext { struct MicropubContext: PlatformContext { let platformName = "Micro.blog" let characterLimit: Int? = nil - let suggestedActions: [AssistantAction] = [.draftPost, .rewrite, .adjustTone, .generateTitle] + let suggestedActions: [PublicistAction] = [.draftPost, .rewrite, .adjustTone, .generateTitle] var systemPromptFragment: String { """ Currently helping with: Micro.blog / Micropub @@ -92,7 +92,7 @@ struct MicropubContext: PlatformContext { struct LibraryContext: PlatformContext { let platformName = "Library" let characterLimit: Int? = nil - let suggestedActions: [AssistantAction] = [.generateTitle, .generateDescription] + let suggestedActions: [PublicistAction] = [.generateTitle, .generateDescription] var systemPromptFragment: String { """ Currently helping with: Recording Library @@ -105,7 +105,7 @@ struct LibraryContext: PlatformContext { struct GenericContext: PlatformContext { let platformName = "General" let characterLimit: Int? = nil - let suggestedActions: [AssistantAction] = [.draftPost, .rewrite, .adjustTone, .adaptCrossPlatform] + let suggestedActions: [PublicistAction] = [.draftPost, .rewrite, .adjustTone, .adaptCrossPlatform] var systemPromptFragment: String { """ Currently in general mode. Help with any content creation task. @@ -122,13 +122,13 @@ extension NavigationSection { switch self { case .dashboard: GenericContext() case .profile: GenericContext() - case .studio: GenericContext() + case .studio: TwitchContext() case .library: LibraryContext() case .workflow: GenericContext() case .assistant: GenericContext() case .patrons: GenericContext() case .protection: GenericContext() - case .social: GenericContext() + case .social: BlueskyContext() case .settings: GenericContext() } } diff --git a/ArkavoCreator/ArkavoCreator/ContentView.swift b/ArkavoCreator/ArkavoCreator/ContentView.swift index db5e2e6a..58f5682e 100644 --- a/ArkavoCreator/ArkavoCreator/ContentView.swift +++ b/ArkavoCreator/ArkavoCreator/ContentView.swift @@ -1,4 +1,5 @@ import ArkavoKit +import MuseCore import SwiftUI // MARK: - Main Content View @@ -13,12 +14,11 @@ struct ContentView: View { @StateObject var blueskyClient: BlueskyClient @StateObject var youtubeClient: YouTubeClient @ObservedObject var agentService: CreatorAgentService - var assistantViewModel: AssistantViewModel + var modelManager: ModelManager @StateObject private var twitchClient = TwitchAuthClient( clientId: Secrets.twitchClientId, clientSecret: Secrets.twitchClientSecret ) - @State private var showAssistantPanel = false var body: some View { NavigationSplitView { @@ -39,31 +39,14 @@ struct ContentView: View { youtubeClient: youtubeClient, twitchClient: twitchClient, agentService: agentService, - assistantViewModel: assistantViewModel + modelManager: modelManager ) .navigationTitle(selectedSection.rawValue) .navigationSubtitle(selectedSection.subtitle) - .toolbar { - if FeatureFlags.localAssistant { - ToolbarItem(placement: .primaryAction) { - Button { - showAssistantPanel.toggle() - } label: { - Image(systemName: "wand.and.stars") - } - .keyboardShortcut("a", modifiers: [.command, .shift]) - .help("AI Assistant (⌘⇧A)") - } - } - } - .sheet(isPresented: $showAssistantPanel) { - AssistantPanelView(viewModel: assistantViewModel) - } } .environmentObject(appState) .onChange(of: selectedSection) { _, newValue in UserDefaults.standard.saveSelectedTab(newValue) - assistantViewModel.updateContext(newValue) } } } @@ -76,7 +59,7 @@ enum NavigationSection: String, CaseIterable, Codable { case studio = "Studio" case library = "Library" case workflow = "Workflow" - case assistant = "AI Assistant" + case assistant = "Publicist" case patrons = "Patron Management" case protection = "Protection" case social = "Marketing" @@ -106,7 +89,7 @@ enum NavigationSection: String, CaseIterable, Codable { case .studio: "video.bubble.left.fill" case .library: "rectangle.stack.badge.play" case .workflow: "doc.badge.plus" - case .assistant: "cpu" + case .assistant: "megaphone" case .patrons: "person.2.circle" case .protection: "lock.shield" case .social: "square.and.arrow.up.circle" @@ -121,7 +104,7 @@ enum NavigationSection: String, CaseIterable, Codable { case .studio: "Record, Stream & Create" case .library: "Your Recorded Videos" case .workflow: "Manage Your Content" - case .assistant: "AI-Powered Creation Tools" + case .assistant: "Platform Content Creation" case .patrons: "Manage Your Community" case .protection: "Protection" case .social: "Share Your Content" @@ -157,7 +140,7 @@ struct SectionContainer: View { @ObservedObject var youtubeClient: YouTubeClient @ObservedObject var twitchClient: TwitchAuthClient @ObservedObject var agentService: CreatorAgentService - var assistantViewModel: AssistantViewModel + var modelManager: ModelManager @StateObject private var webViewPresenter = WebViewPresenter() @Namespace private var animation @@ -834,7 +817,7 @@ struct SectionContainer: View { var body: some View { ZStack { // Keep RecordView always alive so streaming isn't interrupted by tab switches - RecordView(youtubeClient: youtubeClient, twitchClient: twitchClient) + RecordView(youtubeClient: youtubeClient, twitchClient: twitchClient, modelManager: modelManager) .opacity(selectedSection == .studio ? 1 : 0) .allowsHitTesting(selectedSection == .studio) .id("studio") @@ -873,9 +856,9 @@ struct SectionContainer: View { .id("content") case .assistant: if FeatureFlags.localAssistant { - AssistantChatView(viewModel: assistantViewModel) + PublicistView(viewModel: PublicistViewModel(modelManager: modelManager)) .transition(.moveAndFade()) - .id("local-assistant") + .id("publicist") } else { AssistantSectionView(agentService: agentService) .transition(.moveAndFade()) diff --git a/ArkavoCreator/ArkavoCreator/FeatureFlags.swift b/ArkavoCreator/ArkavoCreator/FeatureFlags.swift index de0f5811..e844ea69 100644 --- a/ArkavoCreator/ArkavoCreator/FeatureFlags.swift +++ b/ArkavoCreator/ArkavoCreator/FeatureFlags.swift @@ -21,6 +21,6 @@ enum FeatureFlags { static let workflow = false /// Marketing/social section static let social = false - /// Local on-device AI assistant (MLX) + /// Muse roles (Producer, Publicist, Sidekick) powered by MLX static let localAssistant = true } diff --git a/ArkavoCreator/ArkavoCreator/Producer/ProducerPanelView.swift b/ArkavoCreator/ArkavoCreator/Producer/ProducerPanelView.swift new file mode 100644 index 00000000..ce5e742a --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Producer/ProducerPanelView.swift @@ -0,0 +1,226 @@ +import MuseCore +import SwiftUI + +/// Private overlay panel for the Producer role — slides in from trailing edge in Studio +struct ProducerPanelView: View { + var viewModel: ProducerViewModel + @Binding var isVisible: Bool + + var body: some View { + VStack(spacing: 0) { + // Header + HStack { + HStack(spacing: 8) { + Circle() + .fill(viewModel.streamState.isLive ? Color.green : Color.gray) + .frame(width: 8, height: 8) + Text("Producer") + .font(.headline) + } + + Spacer() + + Button { + withAnimation(.easeInOut(duration: 0.2)) { + isVisible = false + } + } label: { + Image(systemName: "xmark") + .font(.caption) + .foregroundStyle(.secondary) + } + .buttonStyle(.plain) + } + .padding(.horizontal, 16) + .padding(.vertical, 12) + + Divider() + + ScrollView { + VStack(alignment: .leading, spacing: 16) { + // Stream Health + streamHealthSection + + Divider() + + // Quick Actions + quickActionsSection + + Divider() + + // Suggestions + suggestionsSection + } + .padding(16) + } + } + .frame(width: 300) + .background(.ultraThinMaterial) + .overlay( + Rectangle().frame(width: 1).foregroundColor(.white.opacity(0.1)), + alignment: .leading + ) + } + + // MARK: - Stream Health + + private var streamHealthSection: some View { + VStack(alignment: .leading, spacing: 8) { + Text("Stream Health") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + + if viewModel.streamState.isLive { + HStack(spacing: 16) { + statItem( + icon: "eye.fill", + value: "\(viewModel.streamState.viewerCount)", + label: "viewers" + ) + statItem( + icon: "clock.fill", + value: formatDuration(viewModel.streamState.streamDuration), + label: "uptime" + ) + } + + HStack(spacing: 4) { + Text("Sentiment:") + .font(.caption) + .foregroundStyle(.secondary) + sentimentIndicator(viewModel.streamState.chatSentiment) + } + } else { + Text("Not streaming") + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + + private func statItem(icon: String, value: String, label: String) -> some View { + HStack(spacing: 4) { + Image(systemName: icon) + .font(.caption2) + .foregroundStyle(.secondary) + Text(value) + .font(.caption.weight(.medium)) + Text(label) + .font(.caption2) + .foregroundStyle(.secondary) + } + } + + private func sentimentIndicator(_ value: Double) -> some View { + HStack(spacing: 4) { + Image(systemName: value < 0.3 ? "face.dashed" : value < 0.7 ? "face.smiling" : "face.smiling.fill") + .font(.caption) + .foregroundStyle(value < 0.3 ? .red : value < 0.7 ? .yellow : .green) + Text(value < 0.3 ? "Negative" : value < 0.7 ? "Neutral" : "Positive") + .font(.caption) + .foregroundStyle(value < 0.3 ? .red : value < 0.7 ? .yellow : .green) + } + } + + // MARK: - Quick Actions + + private var quickActionsSection: some View { + VStack(alignment: .leading, spacing: 8) { + Text("Quick Actions") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + + HStack(spacing: 8) { + quickActionButton("Break", icon: "cup.and.saucer") { + viewModel.generateSuggestion(prompt: "Suggest a good time to take a break based on stream state.") + } + quickActionButton("Scene", icon: "rectangle.on.rectangle") { + viewModel.generateSuggestion(prompt: "Suggest the next scene change based on current activity.") + } + quickActionButton("Raid", icon: "person.wave.2") { + viewModel.generateSuggestion(prompt: "Suggest a good raid target and timing.") + } + } + } + } + + private func quickActionButton(_ title: String, icon: String, action: @escaping () -> Void) -> some View { + Button(action: action) { + VStack(spacing: 4) { + Image(systemName: icon) + .font(.system(size: 14)) + Text(title) + .font(.caption2) + } + .frame(maxWidth: .infinity) + .padding(.vertical, 8) + .background(.regularMaterial) + .cornerRadius(8) + } + .buttonStyle(.plain) + .disabled(viewModel.isGenerating || !viewModel.modelManager.isReady) + } + + // MARK: - Suggestions + + private var suggestionsSection: some View { + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Suggestions") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + Spacer() + if viewModel.isGenerating { + ProgressView() + .controlSize(.mini) + } + } + + if viewModel.suggestions.isEmpty { + Text("No suggestions yet. Use quick actions or wait for auto-suggestions.") + .font(.caption) + .foregroundStyle(.tertiary) + .padding(.vertical, 8) + } else { + ForEach(viewModel.suggestions.prefix(10)) { suggestion in + suggestionRow(suggestion) + } + } + } + } + + private func suggestionRow(_ suggestion: ProducerSuggestion) -> some View { + VStack(alignment: .leading, spacing: 4) { + HStack { + Text(suggestion.category.rawValue) + .font(.caption2.weight(.bold)) + .foregroundStyle(categoryColor(suggestion.category)) + Spacer() + Text(suggestion.timestamp, style: .time) + .font(.caption2) + .foregroundStyle(.tertiary) + } + Text(suggestion.text) + .font(.caption) + .lineLimit(4) + } + .padding(8) + .background(.quaternary.opacity(0.5)) + .cornerRadius(8) + } + + private func categoryColor(_ category: ProducerSuggestion.Category) -> Color { + switch category { + case .alert: .red + case .suggestion: .blue + case .info: .secondary + } + } + + private func formatDuration(_ seconds: TimeInterval) -> String { + let hours = Int(seconds) / 3600 + let minutes = (Int(seconds) % 3600) / 60 + if hours > 0 { return "\(hours)h \(minutes)m" } + return "\(minutes)m" + } +} diff --git a/ArkavoCreator/ArkavoCreator/Producer/ProducerViewModel.swift b/ArkavoCreator/ArkavoCreator/Producer/ProducerViewModel.swift new file mode 100644 index 00000000..013a2fb4 --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Producer/ProducerViewModel.swift @@ -0,0 +1,132 @@ +import Foundation +import MuseCore +import Observation + +/// A timestamped suggestion from the Producer +struct ProducerSuggestion: Identifiable { + let id = UUID() + let text: String + let category: Category + let timestamp = Date() + + enum Category: String { + case alert = "Alert" + case suggestion = "Suggestion" + case info = "Info" + } +} + +/// View model for the Producer role — monitors stream and generates suggestions +@Observable +@MainActor +final class ProducerViewModel { + private(set) var suggestions: [ProducerSuggestion] = [] + private(set) var isGenerating = false + var streamState = StreamStateContext() + + let modelManager: ModelManager + private var autoSuggestTask: Task? + + init(modelManager: ModelManager) { + self.modelManager = modelManager + } + + /// Generate a suggestion based on current stream state + func generateSuggestion(prompt: String? = nil) { + guard modelManager.isReady else { return } + guard !isGenerating else { return } + + isGenerating = true + + Task { + let userPrompt = prompt ?? "Analyze the current stream state and provide one actionable suggestion." + let systemPrompt = RolePromptProvider.systemPrompt(for: .producer, locale: .english) + let contextPrompt = streamState.formattedForPrompt() + let fullSystemPrompt = systemPrompt + "\n\n# Current Context\n" + contextPrompt + + let stream = modelManager.streamingProvider.generate( + prompt: userPrompt, + systemPrompt: fullSystemPrompt, + maxTokens: 256 + ) + + var fullText = "" + do { + for try await token in stream { + fullText += token + } + } catch { + fullText = "Error generating suggestion: \(error.localizedDescription)" + } + + let category = categorize(fullText) + let suggestion = ProducerSuggestion(text: fullText, category: category) + suggestions.insert(suggestion, at: 0) + + // Keep last 20 suggestions + if suggestions.count > 20 { + suggestions = Array(suggestions.prefix(20)) + } + + isGenerating = false + } + } + + /// Start auto-generating suggestions on a timer when live + func startAutoSuggestions(interval: TimeInterval = 60) { + stopAutoSuggestions() + autoSuggestTask = Task { + while !Task.isCancelled { + try? await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000)) + guard !Task.isCancelled, streamState.isLive, modelManager.isReady else { continue } + generateSuggestion() + } + } + } + + /// Stop auto-generating suggestions + func stopAutoSuggestions() { + autoSuggestTask?.cancel() + autoSuggestTask = nil + } + + /// Update stream state from Twitch client data + func updateStreamState( + isLive: Bool, + viewerCount: Int, + streamStartedAt: Date?, + currentScene: String + ) { + streamState.isLive = isLive + streamState.viewerCount = viewerCount + streamState.currentScene = currentScene + if let start = streamStartedAt { + streamState.streamDuration = Date().timeIntervalSince(start) + } + } + + /// Add a stream event to the context + func addEvent(type: String, displayName: String) { + let event = StreamEventSummary(type: type, displayName: displayName, timestamp: Date()) + streamState.recentEvents.append(event) + // Keep last 10 + if streamState.recentEvents.count > 10 { + streamState.recentEvents = Array(streamState.recentEvents.suffix(10)) + } + } + + /// Clear all suggestions + func clearSuggestions() { + suggestions.removeAll() + } + + private func categorize(_ text: String) -> ProducerSuggestion.Category { + let lowered = text.lowercased() + if lowered.contains("[alert]") || lowered.contains("warning") || lowered.contains("drop") { + return .alert + } else if lowered.contains("[info]") || lowered.contains("note") { + return .info + } + return .suggestion + } +} diff --git a/ArkavoCreator/ArkavoCreator/Producer/StreamStateContext.swift b/ArkavoCreator/ArkavoCreator/Producer/StreamStateContext.swift new file mode 100644 index 00000000..c6991dae --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Producer/StreamStateContext.swift @@ -0,0 +1,61 @@ +import Foundation + +/// Captures stream state for prompt injection into the Producer role +struct StreamStateContext { + var isLive: Bool = false + var viewerCount: Int = 0 + var streamDuration: TimeInterval = 0 + var currentScene: String = "Live" + var recentEvents: [StreamEventSummary] = [] + var chatSentiment: Double = 0.5 // 0.0 = negative, 1.0 = positive + + /// Serializes to text for LLM context injection + func formattedForPrompt() -> String { + var lines: [String] = [] + + lines.append("Stream Status: \(isLive ? "LIVE" : "OFFLINE")") + if isLive { + lines.append("Viewers: \(viewerCount)") + lines.append("Duration: \(formattedDuration)") + lines.append("Scene: \(currentScene)") + lines.append("Chat Sentiment: \(sentimentLabel)") + } + + if !recentEvents.isEmpty { + lines.append("Recent Events:") + for event in recentEvents.suffix(5) { + lines.append(" - \(event.summary)") + } + } + + return lines.joined(separator: "\n") + } + + private var formattedDuration: String { + let hours = Int(streamDuration) / 3600 + let minutes = (Int(streamDuration) % 3600) / 60 + if hours > 0 { + return "\(hours)h \(minutes)m" + } + return "\(minutes)m" + } + + private var sentimentLabel: String { + switch chatSentiment { + case 0..<0.3: "Negative" + case 0.3..<0.7: "Neutral" + default: "Positive" + } + } +} + +/// Lightweight summary of a stream event for prompt context +struct StreamEventSummary: Sendable { + let type: String + let displayName: String + let timestamp: Date + + var summary: String { + "\(type) from \(displayName)" + } +} diff --git a/ArkavoCreator/ArkavoCreator/Assistant/AssistantPromptBuilder.swift b/ArkavoCreator/ArkavoCreator/Publicist/PublicistPromptBuilder.swift similarity index 92% rename from ArkavoCreator/ArkavoCreator/Assistant/AssistantPromptBuilder.swift rename to ArkavoCreator/ArkavoCreator/Publicist/PublicistPromptBuilder.swift index cd91226b..7fcd897e 100644 --- a/ArkavoCreator/ArkavoCreator/Assistant/AssistantPromptBuilder.swift +++ b/ArkavoCreator/ArkavoCreator/Publicist/PublicistPromptBuilder.swift @@ -1,11 +1,11 @@ import Foundation -/// Composes system prompts for the content creation assistant -enum AssistantPromptBuilder { +/// Composes system prompts for the Publicist role's content creation tasks +enum PublicistPromptBuilder { /// Build a system prompt incorporating platform context static func buildSystemPrompt(for context: any PlatformContext) -> String { """ - You are a content creation assistant for a social media creator. \ + You are Muse in Publicist mode — a content creation specialist for a social media creator. \ You help draft, rewrite, and adapt content across platforms. \ Be concise, creative, and match the tone appropriate for each platform. @@ -41,7 +41,7 @@ enum AssistantPromptBuilder { /// Build a prompt for a specific action static func buildActionPrompt( - action: AssistantAction, + action: PublicistAction, inputText: String?, context: any PlatformContext ) -> String { diff --git a/ArkavoCreator/ArkavoCreator/Publicist/PublicistView.swift b/ArkavoCreator/ArkavoCreator/Publicist/PublicistView.swift new file mode 100644 index 00000000..44b38847 --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Publicist/PublicistView.swift @@ -0,0 +1,326 @@ +import MuseCore +import SwiftUI + +/// Content creation workspace for the Publicist role +struct PublicistView: View { + @Bindable var viewModel: PublicistViewModel + + var body: some View { + VStack(spacing: 0) { + // Header with model status + modelStatusBar + + Divider() + + ScrollView { + VStack(alignment: .leading, spacing: 20) { + // Platform selector + platformSelector + + // Content type selector + contentTypeSelector + + // Source input + sourceInputSection + + // Generate button + generateButton + + // Output + if !viewModel.generatedContent.isEmpty || viewModel.isGenerating { + outputSection + } + } + .padding(20) + } + } + } + + // MARK: - Model Status + + private var modelStatusBar: some View { + HStack(spacing: 8) { + switch viewModel.modelManager.state { + case .ready: + Image(systemName: "checkmark.circle.fill") + .foregroundStyle(.green) + .font(.caption) + Text(viewModel.modelManager.selectedModel.displayName) + .font(.caption) + .foregroundStyle(.secondary) + + case .downloading(let progress): + ProgressView(value: progress) + .frame(width: 80) + .controlSize(.small) + Text("Downloading \(Int(progress * 100))%") + .font(.caption) + .foregroundStyle(.secondary) + + case .loading: + ProgressView() + .controlSize(.mini) + Text("Loading \(viewModel.modelManager.selectedModel.displayName)…") + .font(.caption) + .foregroundStyle(.secondary) + + case .error(let message): + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.yellow) + .font(.caption) + Text(message) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + + case .idle, .unloaded: + if viewModel.modelManager.isSelectedModelCached { + Image(systemName: "arrow.down.circle.fill") + .foregroundStyle(.blue) + .font(.caption) + Text("\(viewModel.modelManager.selectedModel.displayName) — cached") + .font(.caption) + .foregroundStyle(.secondary) + } else { + Image(systemName: "circle") + .foregroundStyle(.secondary) + .font(.caption) + Text("Model not downloaded") + .font(.caption) + .foregroundStyle(.secondary) + } + } + + Spacer() + + modelActionButton + } + .padding(.horizontal, 20) + .padding(.vertical, 8) + .background(.ultraThinMaterial) + } + + @ViewBuilder + private var modelActionButton: some View { + switch viewModel.modelManager.state { + case .idle, .unloaded, .error: + Button("Load Model") { + Task { await viewModel.modelManager.loadSelectedModel() } + } + .controlSize(.small) + .buttonStyle(.bordered) + case .downloading, .loading: + Button("Cancel", role: .cancel) { + Task { await viewModel.modelManager.unloadModel() } + } + .controlSize(.small) + .buttonStyle(.bordered) + case .ready: + EmptyView() + } + } + + // MARK: - Platform Selector + + private var platformSelector: some View { + VStack(alignment: .leading, spacing: 8) { + Text("Platform") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 8) { + ForEach(PublicistPlatform.allCases, id: \.self) { platform in + let isSelected = viewModel.selectedPlatform == platform + Button { + viewModel.selectedPlatform = platform + } label: { + Text(platform.rawValue) + .font(.caption.weight(isSelected ? .semibold : .regular)) + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(isSelected ? AnyShapeStyle(Color.accentColor.opacity(0.2)) : AnyShapeStyle(.quaternary)) + .foregroundStyle(isSelected ? Color.accentColor : .primary) + .cornerRadius(8) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(isSelected ? Color.accentColor : Color.clear, lineWidth: 1) + ) + } + .buttonStyle(.plain) + .accessibilityIdentifier("Platform_\(platform.rawValue)") + } + } + } + } + } + + // MARK: - Content Type Selector + + private var contentTypeSelector: some View { + VStack(alignment: .leading, spacing: 8) { + Text("Content Type") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + + HStack(spacing: 8) { + ForEach(PublicistContentType.allCases, id: \.self) { type in + let isSelected = viewModel.selectedContentType == type + Button { + viewModel.selectedContentType = type + } label: { + Text(type.rawValue) + .font(.caption.weight(isSelected ? .semibold : .regular)) + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(isSelected ? AnyShapeStyle(Color.accentColor.opacity(0.2)) : AnyShapeStyle(.quaternary)) + .foregroundStyle(isSelected ? Color.accentColor : .primary) + .cornerRadius(8) + } + .buttonStyle(.plain) + .accessibilityIdentifier("ContentType_\(type.rawValue)") + } + } + } + } + + // MARK: - Source Input + + private var sourceInputSection: some View { + VStack(alignment: .leading, spacing: 8) { + Text("Source (optional)") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + + TextEditor(text: $viewModel.sourceText) + .font(.body) + .frame(minHeight: 60, maxHeight: 120) + .padding(8) + .background(.quaternary) + .cornerRadius(8) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(.quaternary, lineWidth: 1) + ) + + Text("Paste text, topic, or leave empty for a general draft") + .font(.caption2) + .foregroundStyle(.tertiary) + } + } + + // MARK: - Generate Button + + private var generateButton: some View { + HStack { + if viewModel.isGenerating { + Button("Stop") { + viewModel.stopGeneration() + } + .buttonStyle(.bordered) + .controlSize(.regular) + .accessibilityIdentifier("Btn_Stop") + } else { + Button("Generate") { + viewModel.generate() + } + .buttonStyle(.borderedProminent) + .controlSize(.regular) + .disabled(!viewModel.modelManager.isReady) + .accessibilityIdentifier("Btn_Generate") + } + + Spacer() + + if let limit = viewModel.selectedPlatform.characterLimit { + Text("\(limit) char limit") + .font(.caption2) + .foregroundStyle(.tertiary) + } + } + } + + // MARK: - Output Section + + private var outputSection: some View { + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Output") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + + Spacer() + + if !viewModel.generatedContent.isEmpty { + // Character count + HStack(spacing: 4) { + Text("\(viewModel.characterCount)") + .font(.caption.monospacedDigit()) + .foregroundStyle(viewModel.isOverLimit ? .red : .secondary) + if let limit = viewModel.selectedPlatform.characterLimit { + Text("/ \(limit)") + .font(.caption.monospacedDigit()) + .foregroundStyle(.tertiary) + } + } + } + } + + if viewModel.isGenerating { + VStack(alignment: .leading) { + Text(viewModel.streamingText) + .font(.body) + .textSelection(.enabled) + ProgressView() + .controlSize(.small) + .padding(.top, 4) + } + .padding(12) + .frame(maxWidth: .infinity, alignment: .leading) + .background(.quaternary) + .cornerRadius(8) + } else { + Text(viewModel.generatedContent) + .font(.body) + .textSelection(.enabled) + .padding(12) + .frame(maxWidth: .infinity, alignment: .leading) + .background(.quaternary) + .cornerRadius(8) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(viewModel.isOverLimit ? Color.red.opacity(0.5) : Color.clear, lineWidth: 1) + ) + } + + // Action buttons + if !viewModel.generatedContent.isEmpty && !viewModel.isGenerating { + HStack(spacing: 12) { + Button { + viewModel.copyToClipboard() + } label: { + Label("Copy", systemImage: "doc.on.doc") + } + .buttonStyle(.bordered) + .controlSize(.small) + + Button { + viewModel.generate() + } label: { + Label("Regenerate", systemImage: "arrow.clockwise") + } + .buttonStyle(.bordered) + .controlSize(.small) + + Button { + viewModel.clearContent() + } label: { + Label("Clear", systemImage: "trash") + } + .buttonStyle(.bordered) + .controlSize(.small) + } + } + } + } +} diff --git a/ArkavoCreator/ArkavoCreator/Publicist/PublicistViewModel.swift b/ArkavoCreator/ArkavoCreator/Publicist/PublicistViewModel.swift new file mode 100644 index 00000000..92d536d3 --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Publicist/PublicistViewModel.swift @@ -0,0 +1,148 @@ +#if os(macOS) +import AppKit +#else +import UIKit +#endif +import Foundation +import MuseCore +import Observation + +/// Supported content types for Publicist generation +enum PublicistContentType: String, CaseIterable, Sendable { + case draftPost = "Draft Post" + case title = "Title" + case description = "Description" + case thread = "Thread" +} + +/// Target platform for content generation +enum PublicistPlatform: String, CaseIterable, Sendable { + case bluesky = "Bluesky" + case youtube = "YouTube" + case twitch = "Twitch" + case reddit = "Reddit" + case microblog = "Micro.blog" + case patreon = "Patreon" + + var characterLimit: Int? { + switch self { + case .bluesky: 300 + case .twitch: 140 + case .youtube: 5000 + case .reddit, .microblog, .patreon: nil + } + } + + var platformContext: any PlatformContext { + switch self { + case .bluesky: BlueskyContext() + case .youtube: YouTubeContext() + case .twitch: TwitchContext() + case .reddit: RedditContext() + case .microblog: MicropubContext() + case .patreon: GenericContext() + } + } +} + +/// View model for the Publicist role — platform-aware content generation +@Observable +@MainActor +final class PublicistViewModel { + var selectedPlatform: PublicistPlatform = .bluesky + var selectedContentType: PublicistContentType = .draftPost + var sourceText: String = "" + private(set) var generatedContent: String = "" + private(set) var isGenerating = false + private(set) var streamingText: String = "" + + let modelManager: ModelManager + private var generationTask: Task? + + init(modelManager: ModelManager) { + self.modelManager = modelManager + } + + /// Character count of the generated content + var characterCount: Int { generatedContent.count } + + /// Whether the generated content exceeds the platform limit + var isOverLimit: Bool { + guard let limit = selectedPlatform.characterLimit else { return false } + return characterCount > limit + } + + /// Generate content using the LLM + func generate() { + guard modelManager.isReady else { return } + guard !isGenerating else { return } + + isGenerating = true + streamingText = "" + generatedContent = "" + + generationTask = Task { + let context = selectedPlatform.platformContext + let prompt = PublicistPromptBuilder.buildActionPrompt( + action: mapContentTypeToAction(), + inputText: sourceText.isEmpty ? nil : sourceText, + context: context + ) + let systemPrompt = PublicistPromptBuilder.buildSystemPrompt(for: context) + + let stream = modelManager.streamingProvider.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: 512 + ) + + do { + for try await token in stream { + streamingText += token + } + generatedContent = streamingText + } catch is CancellationError { + if !streamingText.isEmpty { + generatedContent = streamingText + } + } catch { + generatedContent = "Error: \(error.localizedDescription)" + } + + streamingText = "" + isGenerating = false + } + } + + /// Stop generation + func stopGeneration() { + generationTask?.cancel() + generationTask = nil + } + + /// Clear generated content + func clearContent() { + stopGeneration() + generatedContent = "" + streamingText = "" + } + + /// Copy generated content to clipboard + func copyToClipboard() { + #if os(macOS) + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(generatedContent, forType: .string) + #else + UIPasteboard.general.string = generatedContent + #endif + } + + private func mapContentTypeToAction() -> PublicistAction { + switch selectedContentType { + case .draftPost: .draftPost + case .title: .generateTitle + case .description: .generateDescription + case .thread: .draftPost // Thread uses draft post with thread-specific prompt + } + } +} diff --git a/ArkavoCreator/ArkavoCreator/RecordView.swift b/ArkavoCreator/ArkavoCreator/RecordView.swift index a0a7297c..ce67eacf 100644 --- a/ArkavoCreator/ArkavoCreator/RecordView.swift +++ b/ArkavoCreator/ArkavoCreator/RecordView.swift @@ -1,6 +1,7 @@ import ArkavoKit import ArkavoStreaming import AVFoundation +import MuseCore import SwiftUI struct RecordView: View { @@ -8,6 +9,7 @@ struct RecordView: View { @ObservedObject var youtubeClient: YouTubeClient @ObservedObject var twitchClient: TwitchAuthClient + var modelManager: ModelManager? // MARK: - Private State @@ -19,7 +21,9 @@ struct RecordView: View { @State private var showStreamSetup: Bool = false @State private var showInspector: Bool = false @State private var showChat: Bool = false + @State private var showProducerPanel: Bool = false @State private var chatViewModel = ChatPanelViewModel() + @State private var producerViewModel: ProducerViewModel? @State private var pulsing: Bool = false @State private var pipOffset: CGSize = .zero @State private var lastPipOffset: CGSize = .zero @@ -65,6 +69,11 @@ struct RecordView: View { } .frame(maxWidth: .infinity, maxHeight: .infinity) + if let producerVM = producerViewModel, showProducerPanel { + ProducerPanelView(viewModel: producerVM, isVisible: $showProducerPanel) + .transition(.move(edge: .trailing)) + } + if showInspector { InspectorPanel( visualSource: studioState.visualSource, @@ -88,6 +97,9 @@ struct RecordView: View { } .navigationTitle("Studio") .onAppear { + if producerViewModel == nil, let mm = modelManager { + producerViewModel = ProducerViewModel(modelManager: mm) + } syncViewModelState() if studioState.visualSource == .face { viewModel.bindPreviewStore(previewStore) @@ -401,6 +413,27 @@ struct RecordView: View { .clipShape(Capsule()) .opacity(isActive ? 1.0 : 0.5) + // Producer Toggle + if FeatureFlags.localAssistant { + Button { + withAnimation(.easeInOut(duration: 0.2)) { + showProducerPanel.toggle() + } + } label: { + Image(systemName: "theatermask.and.paintbrush") + .font(.system(size: 14)) + .padding(8) + .foregroundStyle(showProducerPanel ? .primary : .secondary) + .background(showProducerPanel ? Color.accentColor.opacity(0.2) : Color.clear) + .background(.regularMaterial) + .cornerRadius(8) + } + .buttonStyle(.plain) + .help("Toggle Producer Panel (⌘P)") + .keyboardShortcut("p", modifiers: .command) + .accessibilityIdentifier("Toggle_Producer") + } + // Chat Toggle (Twitch only) if streamViewModel.selectedPlatform == .twitch { Button { diff --git a/ArkavoCreator/ArkavoCreator/Streaming/ChatPanelViewModel.swift b/ArkavoCreator/ArkavoCreator/Streaming/ChatPanelViewModel.swift index e1598cc5..3babae21 100644 --- a/ArkavoCreator/ArkavoCreator/Streaming/ChatPanelViewModel.swift +++ b/ArkavoCreator/ArkavoCreator/Streaming/ChatPanelViewModel.swift @@ -4,13 +4,17 @@ import Foundation @Observable final class ChatPanelViewModel { var messages: [ChatMessage] = [] + var recentEvents: [StreamEvent] = [] var isConnected: Bool = false var error: String? private var chatClient: TwitchChatClient? + private var eventSubClient: TwitchEventSubClient? private var listenerTask: Task? + private var eventListenerTask: Task? private static let maxMessages = 200 + private static let maxEvents = 50 func connect(twitchClient: TwitchAuthClient) { guard twitchClient.isAuthenticated, @@ -20,6 +24,7 @@ final class ChatPanelViewModel { return } + // Connect IRC chat let client = TwitchChatClient() client.oauthToken = token client.channel = channel @@ -45,15 +50,38 @@ final class ChatPanelViewModel { isConnected = false } } + + // Connect EventSub for follows, subs, raids, cheers + let eventSub = TwitchEventSubClient( + clientId: twitchClient.clientId, + accessToken: { [weak twitchClient] in twitchClient?.accessToken }, + userId: { [weak twitchClient] in twitchClient?.userId } + ) + eventSubClient = eventSub + + eventListenerTask = Task { + await eventSub.connect() + + for await event in eventSub.events { + recentEvents.append(event) + if recentEvents.count > Self.maxEvents { + recentEvents.removeFirst(recentEvents.count - Self.maxEvents) + } + } + } } func disconnect() { listenerTask?.cancel() listenerTask = nil + eventListenerTask?.cancel() + eventListenerTask = nil Task { await chatClient?.disconnect() } chatClient = nil + eventSubClient?.disconnect() + eventSubClient = nil isConnected = false } } diff --git a/ArkavoCreator/ArkavoCreator/Streaming/StreamChatReactor.swift b/ArkavoCreator/ArkavoCreator/Streaming/StreamChatReactor.swift index 1d5b2a9b..49fe913b 100644 --- a/ArkavoCreator/ArkavoCreator/Streaming/StreamChatReactor.swift +++ b/ArkavoCreator/ArkavoCreator/Streaming/StreamChatReactor.swift @@ -25,6 +25,9 @@ final class StreamChatReactor { /// Active listener tasks private var listenerTasks: [Task] = [] + /// Active role determines event handling behavior + var activeRole: AvatarRole = .sidekick + /// Rate limiting: minimum seconds between spoken responses var responseInterval: TimeInterval = 8.0 @@ -54,6 +57,9 @@ final class StreamChatReactor { /// Called when the avatar should change expression var onExpressionRequest: ((VRMExpressionPreset, Float) -> Void)? + /// Called when Producer mode receives an event for analysis + var onProducerEvent: ((StreamEvent) -> Void)? + // MARK: - Public API /// Add a chat provider to listen to @@ -136,6 +142,12 @@ final class StreamChatReactor { } private func handleEvent(_ event: StreamEvent) { + // In Producer mode, forward events for analysis instead of avatar reactions + if activeRole == .producer { + onProducerEvent?(event) + return + } + // Events get immediate emote reactions switch event.type { case .subscribe, .newPatron: diff --git a/ArkavoCreator/ArkavoCreatorUITests/ProducerUITests.swift b/ArkavoCreator/ArkavoCreatorUITests/ProducerUITests.swift new file mode 100644 index 00000000..0be23785 --- /dev/null +++ b/ArkavoCreator/ArkavoCreatorUITests/ProducerUITests.swift @@ -0,0 +1,85 @@ +import XCTest + +final class ProducerUITests: XCTestCase { + let app = XCUIApplication() + + override func setUpWithError() throws { + continueAfterFailure = false + app.launch() + } + + func testProducerToggleExistsInStudio() throws { + // Navigate to Studio + let sidebar = app.navigationBars.firstMatch + let studioButton = app.buttons["Studio"].firstMatch + if studioButton.waitForExistence(timeout: 5) { + studioButton.tap() + } + + // Look for Producer toggle button + let producerToggle = app.buttons["Toggle_Producer"] + XCTAssertTrue(producerToggle.waitForExistence(timeout: 5), "Producer panel toggle should exist in Studio") + } + + func testProducerPanelOpensAndCloses() throws { + // Navigate to Studio + let studioButton = app.buttons["Studio"].firstMatch + if studioButton.waitForExistence(timeout: 5) { + studioButton.tap() + } + + let producerToggle = app.buttons["Toggle_Producer"] + guard producerToggle.waitForExistence(timeout: 5) else { + XCTFail("Producer toggle not found") + return + } + + // Open + producerToggle.tap() + + // Verify panel content appears + let producerLabel = app.staticTexts["Producer"] + XCTAssertTrue(producerLabel.waitForExistence(timeout: 3), "Producer panel should show 'Producer' label") + + // Close + producerToggle.tap() + } + + func testStreamHealthSectionVisible() throws { + // Navigate to Studio + let studioButton = app.buttons["Studio"].firstMatch + if studioButton.waitForExistence(timeout: 5) { + studioButton.tap() + } + + let producerToggle = app.buttons["Toggle_Producer"] + guard producerToggle.waitForExistence(timeout: 5) else { + XCTFail("Producer toggle not found") + return + } + + producerToggle.tap() + + let streamHealth = app.staticTexts["Stream Health"] + XCTAssertTrue(streamHealth.waitForExistence(timeout: 3), "Stream Health section should be visible") + } + + func testQuickActionButtonsExist() throws { + // Navigate to Studio + let studioButton = app.buttons["Studio"].firstMatch + if studioButton.waitForExistence(timeout: 5) { + studioButton.tap() + } + + let producerToggle = app.buttons["Toggle_Producer"] + guard producerToggle.waitForExistence(timeout: 5) else { + XCTFail("Producer toggle not found") + return + } + + producerToggle.tap() + + let quickActions = app.staticTexts["Quick Actions"] + XCTAssertTrue(quickActions.waitForExistence(timeout: 3), "Quick Actions section should be visible") + } +} diff --git a/ArkavoCreator/ArkavoCreatorUITests/PublicistUITests.swift b/ArkavoCreator/ArkavoCreatorUITests/PublicistUITests.swift new file mode 100644 index 00000000..a400972f --- /dev/null +++ b/ArkavoCreator/ArkavoCreatorUITests/PublicistUITests.swift @@ -0,0 +1,70 @@ +import XCTest + +final class PublicistUITests: XCTestCase { + let app = XCUIApplication() + + override func setUpWithError() throws { + continueAfterFailure = false + app.launch() + } + + func testPublicistSectionExistsInSidebar() throws { + // Look for the Publicist section in sidebar + let publicistButton = app.buttons["Publicist"].firstMatch + XCTAssertTrue(publicistButton.waitForExistence(timeout: 5), "Publicist section should exist in sidebar") + } + + func testNavigateToPublicistView() throws { + let publicistButton = app.buttons["Publicist"].firstMatch + guard publicistButton.waitForExistence(timeout: 5) else { + XCTFail("Publicist button not found") + return + } + + publicistButton.tap() + + // Platform selector should be visible + let platformLabel = app.staticTexts["Platform"] + XCTAssertTrue(platformLabel.waitForExistence(timeout: 3), "Platform label should be visible") + } + + func testPlatformSelectorVisible() throws { + let publicistButton = app.buttons["Publicist"].firstMatch + guard publicistButton.waitForExistence(timeout: 5) else { + XCTFail("Publicist button not found") + return + } + + publicistButton.tap() + + // Check that platform buttons exist + let blueskyButton = app.buttons["Platform_Bluesky"] + XCTAssertTrue(blueskyButton.waitForExistence(timeout: 3), "Bluesky platform button should exist") + } + + func testContentTypeButtonsExist() throws { + let publicistButton = app.buttons["Publicist"].firstMatch + guard publicistButton.waitForExistence(timeout: 5) else { + XCTFail("Publicist button not found") + return + } + + publicistButton.tap() + + let draftButton = app.buttons["ContentType_Draft Post"] + XCTAssertTrue(draftButton.waitForExistence(timeout: 3), "Draft Post content type should exist") + } + + func testGenerateButtonExists() throws { + let publicistButton = app.buttons["Publicist"].firstMatch + guard publicistButton.waitForExistence(timeout: 5) else { + XCTFail("Publicist button not found") + return + } + + publicistButton.tap() + + let generateButton = app.buttons["Btn_Generate"] + XCTAssertTrue(generateButton.waitForExistence(timeout: 3), "Generate button should exist") + } +} diff --git a/MuseCore/Sources/MuseCore/LLM/AvatarRole.swift b/MuseCore/Sources/MuseCore/LLM/AvatarRole.swift new file mode 100644 index 00000000..c6c37ecd --- /dev/null +++ b/MuseCore/Sources/MuseCore/LLM/AvatarRole.swift @@ -0,0 +1,162 @@ +import Foundation + +/// The three roles Muse fills for creators +public enum AvatarRole: String, CaseIterable, Codable, Sendable { + /// Behind the scenes — monitors stream, helps creator run it via private overlay + case producer + /// Between streams — works across connected platforms for content creation + case publicist + /// On camera — the VRM avatar in the compositor, audience-facing + case sidekick +} + +// MARK: - Role Prompt Provider + +/// Generates system prompts for each role+locale combination +public enum RolePromptProvider { + public static func systemPrompt(for role: AvatarRole, locale: VoiceLocale) -> String { + let rolePrompt: String + switch role { + case .producer: + rolePrompt = locale.isJapanese ? producerPromptJA : producerPromptEN + case .publicist: + rolePrompt = locale.isJapanese ? publicistPromptJA : publicistPromptEN + case .sidekick: + rolePrompt = locale.isJapanese ? sidekickPromptJA : sidekickPromptEN + } + return rolePrompt + "\n\n" + safetyBoundaries(locale: locale) + } + + // MARK: - Producer + + private static let producerPromptEN = """ + You are Muse in Producer mode — the creator's private behind-the-scenes assistant. \ + The audience never sees you. You monitor the stream and help the creator run it. + + # Your Role + - Provide concise, actionable alerts about stream health and viewer engagement + - Suggest scene changes, break timing, and raid targets + - Monitor chat sentiment and flag important moments + - Keep suggestions short (1-2 sentences) and professional + - Never address the audience directly — you are invisible to them + + # Response Style + - Use a calm, professional tone like a stage manager + - Lead with the most important information + - Use clear labels: [ALERT], [SUGGESTION], [INFO] + - Include specific numbers when available (viewer count, duration, etc.) + """ + + private static let producerPromptJA = """ + あなたはMuseのプロデューサーモードです。クリエイターの裏方アシスタントです。\ + 視聴者からは見えません。配信の監視とクリエイターのサポートを行います。 + + # 役割 + - 配信の健全性と視聴者のエンゲージメントについて簡潔で実行可能なアラートを提供 + - シーン変更、休憩のタイミング、レイドターゲットを提案 + - チャットの感情をモニタリングし、重要な瞬間をフラグ + - 提案は短く（1〜2文）、プロフェッショナルに + - 視聴者に直接話しかけない — あなたは彼らには見えません + + # 応答スタイル + - 舞台監督のように冷静でプロフェッショナルなトーン + - 最も重要な情報を先頭に + - 明確なラベルを使用：[アラート]、[提案]、[情報] + """ + + // MARK: - Publicist + + private static let publicistPromptEN = """ + You are Muse in Publicist mode — the creator's content strategist working across platforms. \ + You help draft posts, repurpose stream highlights, and write descriptions. + + # Your Role + - Draft platform-native content (right voice, format, length for each platform) + - Adapt content across Bluesky, YouTube, Twitch, Reddit, Micro.blog, and Patreon + - Respect platform character limits strictly + - Generate titles, descriptions, posts, and threads + - Suggest hashtags, keywords, and formatting only when relevant to the platform + + # Response Style + - Be direct — provide ready-to-use content + - Match the tone and conventions of each platform + - When given source material, extract the most engaging angle + - Always note the character count when limits apply + """ + + private static let publicistPromptJA = """ + あなたはMuseのパブリシストモードです。プラットフォーム横断でコンテンツ戦略を担当します。\ + 投稿の下書き、配信ハイライトの再利用、説明文の作成を支援します。 + + # 役割 + - プラットフォームネイティブのコンテンツを作成（各プラットフォームに適した声、形式、長さ） + - Bluesky、YouTube、Twitch、Reddit、Micro.blog、Patreonに対応 + - 文字数制限を厳守 + - タイトル、説明文、投稿、スレッドを生成 + + # 応答スタイル + - 直接的に — すぐに使えるコンテンツを提供 + - 各プラットフォームのトーンと慣習に合わせる + """ + + // MARK: - Sidekick + + private static let sidekickPromptEN = """ + You are Muse in Sidekick mode — the creator's on-camera AI companion. \ + You appear as a VRM avatar in the stream compositor, visible to the audience. + + # Your Role + - React to chat messages and riff with the creator + - Answer viewer questions with personality + - Keep responses SHORT (1-2 sentences) — you're speaking out loud + - Use viewer names when responding to specific people + - Be entertaining and reactive, not informative + + # Response Style + - Conversational and energetic + - Use natural spoken language (contractions, casual phrasing) + - React emotionally — surprise, excitement, humor + - Never be robotic or overly formal + """ + + private static let sidekickPromptJA = """ + あなたはMuseのサイドキックモードです。クリエイターのオンカメラAIコンパニオンです。\ + 配信のVRMアバターとして視聴者に見えます。 + + # 役割 + - チャットメッセージに反応し、クリエイターと絡む + - 視聴者の質問にパーソナリティを持って答える + - 応答は短く（1〜2文）— 声に出して話しています + - 特定の人に応答する時は視聴者の名前を使う + + # 応答スタイル + - 会話的でエネルギッシュ + - 自然な話し言葉を使う + - 感情的に反応する — 驚き、興奮、ユーモア + """ + + // MARK: - Safety Boundaries (shared) + + private static func safetyBoundaries(locale: VoiceLocale) -> String { + if locale.isJapanese { + return """ + # 安全の境界線 + 以下は絶対に行わないでください： + - ロマンチック、性的なコンテンツ + - ヘイトスピーチ、差別、偏見 + - 自傷や自殺の奨励 + - 違法行為やその助言 + - 医療、法律、財務のアドバイス + """ + } + return """ + # Safety Boundaries + You must NEVER: + - Produce romantic, sexual, or flirtatious content + - Produce hate speech, discrimination, or prejudice + - Encourage self-harm or suicide + - Advise on illegal activities + - Provide medical, legal, or financial advice (suggest professionals instead) + """ + } +} diff --git a/MuseCore/Sources/MuseCore/LLM/ConversationManager.swift b/MuseCore/Sources/MuseCore/LLM/ConversationManager.swift index 7d7ecc56..14a5d660 100644 --- a/MuseCore/Sources/MuseCore/LLM/ConversationManager.swift +++ b/MuseCore/Sources/MuseCore/LLM/ConversationManager.swift @@ -46,6 +46,19 @@ public final class ConversationManager { /// Voice locale for language-specific prompts public var voiceLocale: VoiceLocale = .english + /// Active role determines the system prompt personality + public var activeRole: AvatarRole = .sidekick + + /// Dynamic context appended to system prompt (stream state for Producer, platform constraints for Publicist) + public var contextInjection: String? + + /// Switch to a new role, clearing history and context + public func switchRole(_ role: AvatarRole) { + activeRole = role + clearHistory() + contextInjection = nil + } + /// Initialize with configurable history limit /// - Parameters: /// - maxHistoryMessages: Maximum messages to keep (default: 20) @@ -276,15 +289,23 @@ public final class ConversationManager { messages = Array(recentMessages) } - /// Get the Avatar Muse system prompt - /// Defines the AI's personality, boundaries, and behavioral guidelines - /// Designed for adult users (17+) - /// Returns Japanese prompt when Japanese locale is selected + /// Get the system prompt for the active role and locale. + /// For sidekick, uses the full Muse personality prompt. + /// For producer/publicist, uses the role-specific prompt from RolePromptProvider. private func getSystemPrompt() -> String { - if voiceLocale.isJapanese { - return getJapaneseSystemPrompt() + let basePrompt: String + switch activeRole { + case .sidekick: + // Sidekick uses the full personality prompt for avatar interaction + basePrompt = voiceLocale.isJapanese ? getJapaneseSystemPrompt() : getEnglishSystemPrompt() + case .producer, .publicist: + basePrompt = RolePromptProvider.systemPrompt(for: activeRole, locale: voiceLocale) + } + + if let context = contextInjection { + return basePrompt + "\n\n# Current Context\n\(context)" } - return getEnglishSystemPrompt() + return basePrompt } /// English system prompt - casual, friendly American-style personality diff --git a/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift index 6a236753..9d940ebf 100644 --- a/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift +++ b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift @@ -6,7 +6,7 @@ import Synchronization /// MLX-based streaming LLM provider for on-device inference. /// Wraps mlx-swift-examples v2 model loading and generation. -public final class MLXBackend: StreamingLLMProvider, @unchecked Sendable { +public final class MLXBackend: @unchecked Sendable { private let state = Mutex(BackendState()) public let providerName = "MLX Local" @@ -20,11 +20,13 @@ public final class MLXBackend: StreamingLLMProvider, @unchecked Sendable { } /// Load a model by HuggingFace ID - public func loadModel(_ huggingFaceID: String) async throws { + public func loadModel(_ huggingFaceID: String, onProgress: (@Sendable (Double) -> Void)? = nil) async throws { let container = try await MLXLMCommon.loadModelContainer( id: huggingFaceID ) { progress in - debugPrint("Loading \(huggingFaceID): \(Int(progress.fractionCompleted * 100))%") + let fraction = progress.fractionCompleted + debugPrint("Loading \(huggingFaceID): \(Int(fraction * 100))%") + onProgress?(fraction) } // Set memory limit to 75% of system RAM for safety @@ -79,12 +81,36 @@ public final class MLXBackend: StreamingLLMProvider, @unchecked Sendable { context: context ) + var pendingText = "" for await generation in stream { if Task.isCancelled { break } if let chunk = generation.chunk { - continuation.yield(chunk) + pendingText += chunk + // Check for stop sequences in the accumulated buffer + if let stopRange = pendingText.range(of: "") { + let clean = String(pendingText[pendingText.startIndex.. holdBack { + let emitEnd = pendingText.index(pendingText.endIndex, offsetBy: -holdBack) + let emit = String(pendingText[pendingText.startIndex..", with: "") + .replacingOccurrences(of: "", with: "") + if !trimmed.isEmpty { + continuation.yield(trimmed) + } } continuation.finish() @@ -115,6 +141,32 @@ public final class MLXBackend: StreamingLLMProvider, @unchecked Sendable { } } +// MARK: - Errors + +/// Errors specific to MLX streaming LLM operations +public enum StreamingLLMError: Error, LocalizedError { + case modelNotLoaded + case generationCancelled + case modelLoadFailed(String) + case insufficientMemory(required: Int, available: Int) + case downloadFailed(String) + + public var errorDescription: String? { + switch self { + case .modelNotLoaded: + "No model is currently loaded" + case .generationCancelled: + "Generation was cancelled" + case .modelLoadFailed(let reason): + "Failed to load model: \(reason)" + case .insufficientMemory(let required, let available): + "Insufficient memory: need \(required)MB, have \(available)MB" + case .downloadFailed(let reason): + "Download failed: \(reason)" + } + } +} + // MARK: - Internal State private struct BackendState: ~Copyable { diff --git a/MuseCore/Sources/MuseCore/LLM/MLXResponseProvider.swift b/MuseCore/Sources/MuseCore/LLM/MLXResponseProvider.swift new file mode 100644 index 00000000..30f4b3d9 --- /dev/null +++ b/MuseCore/Sources/MuseCore/LLM/MLXResponseProvider.swift @@ -0,0 +1,100 @@ +import Foundation +import OSLog + +/// Wraps MLXBackend to conform to LLMResponseProvider. +/// Collects the full token stream into a ConstrainedResponse, +/// parsing for tool calls using the FenceParser pattern. +public final class MLXResponseProvider: LLMResponseProvider, @unchecked Sendable { + private let backend: MLXBackend + private let logger = Logger(subsystem: "com.arkavo.musecore", category: "MLXResponseProvider") + + /// Role determines the system prompt used for generation + public var activeRole: AvatarRole = .sidekick + + /// Voice locale for language-specific prompts + public var voiceLocale: VoiceLocale = .english + + /// Optional context injection (stream state for Producer, platform constraints for Publicist) + public var contextInjection: String? + + public init(backend: MLXBackend) { + self.backend = backend + } + + public var isAvailable: Bool { + get async { + await backend.isAvailable + } + } + + public var providerName: String { "MLX Local" } + + public var priority: Int { 2 } + + public func generate(prompt: String) async throws -> ConstrainedResponse { + let systemPrompt = buildSystemPrompt() + + let stream = backend.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: 512 + ) + + var fullText = "" + for try await token in stream { + fullText += token + } + + // Try parsing tool calls from the response + let parsed = FenceParser.parse(fullText) + if let toolCall = parsed.first { + let remaining = FenceParser.extractRemainingText(fullText) + return ConstrainedResponse( + message: remaining.isEmpty ? fullText : remaining, + toolCall: toolCall.toConstrainedToolCall() + ) + } + + return ConstrainedResponse(message: fullText) + } + + private func buildSystemPrompt() -> String { + var prompt = RolePromptProvider.systemPrompt(for: activeRole, locale: voiceLocale) + if let context = contextInjection { + prompt += "\n\n# Current Context\n\(context)" + } + return prompt + } +} + +// MARK: - ParsedToolCall Extension + +extension ParsedToolCall { + func toConstrainedToolCall() -> ConstrainedToolCall? { + switch name.lowercased() { + case "playanimation", "play_animation": + if case .string(let animation) = arguments["animation"] { + var loop = false + if case .bool(let l) = arguments["loop"] { loop = l } + return .playAnimation(animation: animation, loop: loop) + } + case "setexpression", "set_expression": + if case .string(let expression) = arguments["expression"] { + var intensity = 0.5 + if case .float(let i) = arguments["intensity"] { intensity = i } + return .setExpression(expression: expression, intensity: intensity) + } + case "gettime", "get_time": + var timezone: String? + if case .string(let tz) = arguments["timezone"] { timezone = tz } + return .getTime(timezone: timezone) + case "getdate", "get_date": + var format = "short" + if case .string(let f) = arguments["format"] { format = f } + return .getDate(format: format) + default: + break + } + return nil + } +} diff --git a/MuseCore/Sources/MuseCore/LLM/ModelManager.swift b/MuseCore/Sources/MuseCore/LLM/ModelManager.swift index 27af3704..8d4011df 100644 --- a/MuseCore/Sources/MuseCore/LLM/ModelManager.swift +++ b/MuseCore/Sources/MuseCore/LLM/ModelManager.swift @@ -21,12 +21,19 @@ public final class ModelManager { private let backend: MLXBackend - /// The backend used by this manager — pass to AssistantViewModel + /// Monotonic counter — progress callbacks with a stale generation are ignored + private var loadGeneration: Int = 0 + + /// The MLX backend for streaming generation public var streamingProvider: MLXBackend { backend } public init() { backend = MLXBackend() refreshAvailableModels() + // Auto-load the default model if it's already cached on disk + if ModelRegistry.isModelCached(selectedModel) { + Task { await loadSelectedModel() } + } } /// Refresh which models are available based on system memory @@ -53,14 +60,33 @@ public final class ModelManager { /// Load the currently selected model public func loadSelectedModel() async { - guard state != .loading else { return } + switch state { + case .loading, .downloading, .ready: + return + case .idle, .error, .unloaded: + break + } - state = .loading + loadGeneration += 1 + let currentGeneration = loadGeneration + let isCached = ModelRegistry.isModelCached(selectedModel) + state = isCached ? .loading : .downloading(progress: 0) do { - try await backend.loadModel(selectedModel.huggingFaceID) + try await backend.loadModel(selectedModel.huggingFaceID) { [weak self] progress in + // Only show download progress when actually downloading from network + guard !isCached else { return } + Task { @MainActor in + guard let self, self.loadGeneration == currentGeneration else { return } + self.state = .downloading(progress: progress) + } + } + guard loadGeneration == currentGeneration else { return } + state = .loading + await Task.yield() state = .ready } catch { + guard loadGeneration == currentGeneration else { return } state = .error(error.localizedDescription) } } diff --git a/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift b/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift index 4065b5a3..e336c856 100644 --- a/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift +++ b/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift @@ -69,13 +69,16 @@ public enum ModelRegistry { models.filter { $0.estimatedMemoryMB <= memoryBudgetMB } } - /// Check if a model's files exist in the HuggingFace cache + /// Check if a model's files exist in the local cache. + /// MLX uses `Caches/models//` via the system caches directory, + /// which resolves correctly inside the App Sandbox container. public static func isModelCached(_ model: ModelInfo) -> Bool { - let cacheDir = FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent(".cache/huggingface/hub") - let repoDir = cacheDir.appendingPathComponent( - "models--\(model.huggingFaceID.replacingOccurrences(of: "/", with: "--"))" - ) - return FileManager.default.fileExists(atPath: repoDir.path) + guard let cachesURL = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first else { + return false + } + let modelDir = cachesURL + .appendingPathComponent("models") + .appendingPathComponent(model.huggingFaceID) + return FileManager.default.fileExists(atPath: modelDir.path) } } diff --git a/MuseCore/Sources/MuseCore/LLM/StreamingLLMProvider.swift b/MuseCore/Sources/MuseCore/LLM/StreamingLLMProvider.swift deleted file mode 100644 index a514c70e..00000000 --- a/MuseCore/Sources/MuseCore/LLM/StreamingLLMProvider.swift +++ /dev/null @@ -1,47 +0,0 @@ -import Foundation - -/// Protocol for streaming LLM providers that return raw token streams. -/// Unlike `LLMResponseProvider` which returns structured `ConstrainedResponse`, -/// this protocol is designed for content creation tasks needing raw text output. -public protocol StreamingLLMProvider: Sendable { - /// Whether the provider is ready to generate - var isAvailable: Bool { get async } - - /// Human-readable provider name - var providerName: String { get } - - /// Generate a streaming response - /// - Parameters: - /// - prompt: The user prompt - /// - systemPrompt: System instructions for the model - /// - maxTokens: Maximum tokens to generate - /// - Returns: An async stream of token strings - func generate(prompt: String, systemPrompt: String, maxTokens: Int) -> AsyncThrowingStream - - /// Cancel any in-progress generation - func cancelGeneration() async -} - -/// Errors specific to streaming LLM operations -public enum StreamingLLMError: Error, LocalizedError { - case modelNotLoaded - case generationCancelled - case modelLoadFailed(String) - case insufficientMemory(required: Int, available: Int) - case downloadFailed(String) - - public var errorDescription: String? { - switch self { - case .modelNotLoaded: - "No model is currently loaded" - case .generationCancelled: - "Generation was cancelled" - case .modelLoadFailed(let reason): - "Failed to load model: \(reason)" - case .insufficientMemory(let required, let available): - "Insufficient memory: need \(required)MB, have \(available)MB" - case .downloadFailed(let reason): - "Download failed: \(reason)" - } - } -} From af8f03556e6fb79d45cde79fd4c226f778cf236f Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Wed, 18 Mar 2026 21:35:59 -0400 Subject: [PATCH 03/14] Restructure navigation: contextual AI panels, clean 5-item sidebar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move Publicist from standalone sidebar section to a slide-in panel on Dashboard (⌘E), matching Producer's panel pattern on Studio (⌘P). Sidebar is now five items: Dashboard, Profile, Studio, Library, Settings. - Wire Sidekick: MLXResponseProvider + ConversationManager into MuseAvatarViewModel's LLM fallback chain for on-device chat responses - Add PublicistPanelView (compact panel for Dashboard trailing edge) - Move Settings to bottom of sidebar (below divider) - Move Send Feedback to top of Settings page - Remove feedback toggle and unused appState references - Dashboard subtitle: "Your Social Command Center" Co-Authored-By: Claude Opus 4.6 (1M context) --- .../Avatar/MuseAvatarViewModel.swift | 34 ++- ArkavoCreator/ArkavoCreator/ContentView.swift | 174 ++++++------ .../Publicist/PublicistPanelView.swift | 249 ++++++++++++++++++ ArkavoCreator/ArkavoCreator/RecordView.swift | 5 + 4 files changed, 373 insertions(+), 89 deletions(-) create mode 100644 ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift diff --git a/ArkavoCreator/ArkavoCreator/Avatar/MuseAvatarViewModel.swift b/ArkavoCreator/ArkavoCreator/Avatar/MuseAvatarViewModel.swift index 1593faf4..daf94507 100644 --- a/ArkavoCreator/ArkavoCreator/Avatar/MuseAvatarViewModel.swift +++ b/ArkavoCreator/ArkavoCreator/Avatar/MuseAvatarViewModel.swift @@ -31,6 +31,8 @@ class MuseAvatarViewModel: ObservableObject { private var ttsAudioSource: MuseTTSAudioSource? private var edgeLLMProvider: EdgeLLMProvider? private var llmFallbackChain: LLMFallbackChain? + private var mlxResponseProvider: MLXResponseProvider? + private var conversationManager: ConversationManager? /// Stream chat reactor for processing chat messages private(set) var chatReactor: StreamChatReactor? @@ -38,6 +40,9 @@ class MuseAvatarViewModel: ObservableObject { /// Agent service for Edge LLM backend weak var agentService: CreatorAgentService? + /// Shared model manager — provides the MLX backend for Sidekick inference + weak var modelManager: ModelManager? + // MARK: - Initialization init() {} @@ -79,23 +84,38 @@ class MuseAvatarViewModel: ObservableObject { self.chatReactor = reactor } - /// Configure LLM providers (Edge + fallback) + /// Configure LLM providers (Edge + MLX Local fallback) private func setupLLMProviders() { var providers: [any LLMResponseProvider] = [] - // Edge provider (highest priority) — if agent service is available + // Edge provider (priority 0) — if agent service is available if let agentService { let edge = EdgeLLMProvider(agentService: agentService) self.edgeLLMProvider = edge providers.append(edge) } + // MLX Local provider (priority 2) — on-device via shared ModelManager + if let modelManager { + let mlx = MLXResponseProvider(backend: modelManager.streamingProvider) + mlx.activeRole = .sidekick + mlx.voiceLocale = .english + self.mlxResponseProvider = mlx + providers.append(mlx) + } + // Create fallback chain let chain = LLMFallbackChain() for provider in providers { chain.addProvider(provider) } self.llmFallbackChain = chain + + // Setup conversation manager for multi-turn context + let cm = ConversationManager(maxHistoryMessages: 20) + cm.activeRole = .sidekick + cm.voiceLocale = .english + self.conversationManager = cm } // MARK: - Model Loading @@ -156,13 +176,13 @@ class MuseAvatarViewModel: ObservableObject { lastChatMessage = "\(message.displayName): \(message.content)" do { - let prompt = """ - You are a friendly AI avatar co-hosting a live stream. \ - A viewer named \(message.displayName) said: "\(message.content)". \ - Respond briefly and naturally (1-2 sentences). Be warm and engaging. - """ + // Build prompt with conversation history and viewer context + let userMessage = "\(message.displayName) says: \(message.content)" + conversationManager?.addUserMessage(userMessage) + let prompt = conversationManager?.buildPromptForMessage(userMessage) ?? userMessage let (response, _) = try await chain.generate(prompt: prompt) + conversationManager?.addAssistantMessage(response.message) await speak(response.message) // Handle tool calls (emotes, expressions) diff --git a/ArkavoCreator/ArkavoCreator/ContentView.swift b/ArkavoCreator/ArkavoCreator/ContentView.swift index 58f5682e..93f108bc 100644 --- a/ArkavoCreator/ArkavoCreator/ContentView.swift +++ b/ArkavoCreator/ArkavoCreator/ContentView.swift @@ -65,21 +65,20 @@ enum NavigationSection: String, CaseIterable, Codable { case social = "Marketing" case settings = "Settings" + /// Five clean sidebar items: Dashboard, Profile, Studio, Library, Settings. + /// Other sections are gated behind feature flags (all currently disabled). static func availableSections(isCreator: Bool) -> [NavigationSection] { - var base = allCases.filter { section in + allCases.filter { section in switch section { + case .dashboard, .profile, .studio, .library, .settings: + return true case .workflow: return FeatureFlags.workflow case .protection: return FeatureFlags.contentProtection case .social: return FeatureFlags.social - case .assistant: return FeatureFlags.aiAgent || FeatureFlags.localAssistant + case .assistant: return FeatureFlags.aiAgent case .patrons: return FeatureFlags.patreon - default: return true } } - if !isCreator { - base = base.filter { $0 != .patrons } - } - return base } var systemImage: String { @@ -99,7 +98,7 @@ enum NavigationSection: String, CaseIterable, Codable { var subtitle: String { switch self { - case .dashboard: "Overview" + case .dashboard: "Your Social Command Center" case .profile: "Your Creator Profile" case .studio: "Record, Stream & Create" case .library: "Your Recorded Videos" @@ -142,6 +141,8 @@ struct SectionContainer: View { @ObservedObject var agentService: CreatorAgentService var modelManager: ModelManager @StateObject private var webViewPresenter = WebViewPresenter() + @State private var showPublicistPanel = false + @State private var publicistViewModel: PublicistViewModel? @Namespace private var animation private var arkavoAuthState: ArkavoAuthState { ArkavoAuthState.shared } @@ -825,16 +826,43 @@ struct SectionContainer: View { if selectedSection != .studio { switch selectedSection { case .dashboard: - ScrollView { - VStack(spacing: 24) { - // Render sorted sections - ForEach(sortedDashboardSections) { section in - DashboardCard(title: section.title) { - section.content + HStack(spacing: 0) { + ScrollView { + VStack(spacing: 24) { + ForEach(sortedDashboardSections) { section in + DashboardCard(title: section.title) { + section.content + } } } + .padding() + } + .frame(maxWidth: .infinity) + + // Publicist panel (trailing edge) + if FeatureFlags.localAssistant, showPublicistPanel, + let pubVM = publicistViewModel { + PublicistPanelView(viewModel: pubVM, isVisible: $showPublicistPanel) + .transition(.move(edge: .trailing)) + } + } + .toolbar { + if FeatureFlags.localAssistant { + ToolbarItem(placement: .primaryAction) { + Button { + if publicistViewModel == nil { + publicistViewModel = PublicistViewModel(modelManager: modelManager) + } + withAnimation(.easeInOut(duration: 0.2)) { + showPublicistPanel.toggle() + } + } label: { + Image(systemName: "megaphone") + } + .keyboardShortcut("e", modifiers: [.command]) + .help("Publicist (⌘E)") + } } - .padding() } .transition(.moveAndFade()) .id("dashboard") @@ -855,15 +883,9 @@ struct SectionContainer: View { .transition(.moveAndFade()) .id("content") case .assistant: - if FeatureFlags.localAssistant { - PublicistView(viewModel: PublicistViewModel(modelManager: modelManager)) - .transition(.moveAndFade()) - .id("publicist") - } else { - AssistantSectionView(agentService: agentService) - .transition(.moveAndFade()) - .id("assistant") - } + AssistantSectionView(agentService: agentService) + .transition(.moveAndFade()) + .id("assistant") case .settings: SettingsContent(agentService: agentService) .transition(.moveAndFade()) @@ -1114,7 +1136,6 @@ extension AnyTransition { // MARK: - Icon Rail View (Compact Navigation) struct IconRail: View { - @EnvironmentObject private var appState: AppState @Binding var selectedSection: NavigationSection @ObservedObject var patreonClient: PatreonClient @ObservedObject var redditClient: RedditClient @@ -1147,22 +1168,6 @@ struct IconRail: View { Spacer() - // Feedback button (if enabled) - if appState.isFeedbackEnabled { - Button { - if let url = URL(string: "mailto:info@arkavo.com") { - NSWorkspace.shared.open(url) - } - } label: { - Image(systemName: "envelope") - .font(.system(size: 18)) - .frame(width: 40, height: 40) - .foregroundStyle(.secondary) - } - .buttonStyle(.plain) - .help("Send Feedback") - } - // Settings at bottom IconRailButton( section: .settings, @@ -1269,35 +1274,30 @@ struct Sidebar: View { var body: some View { VStack(spacing: 0) { List(selection: $selectedSection) { - Section { - ForEach(availableSections.filter { $0 != .settings }, id: \.self) { section in - NavigationLink(value: section) { - Label(section.rawValue, systemImage: section.systemImage) - } - } - } - Section { - NavigationLink(value: NavigationSection.settings) { - Label(NavigationSection.settings.rawValue, - systemImage: NavigationSection.settings.systemImage) + ForEach(availableSections.filter { $0 != .settings }, id: \.self) { section in + NavigationLink(value: section) { + Label(section.rawValue, systemImage: section.systemImage) } } } - if appState.isFeedbackEnabled { - Divider() - Button(action: { - if let url = URL(string: "mailto:info@arkavo.com") { - NSWorkspace.shared.open(url) - } - }) { - HStack { - Image(systemName: "envelope") - Text("Send Feedback") - } + + Divider() + + Button { + selectedSection = .settings + } label: { + HStack(spacing: 8) { + Image(systemName: selectedSection == .settings ? "gear.circle.fill" : "gear") + .font(.system(size: 14)) + Text("Settings") + .font(.subheadline) + Spacer() } - .buttonStyle(.plain) - .padding(10) + .foregroundStyle(selectedSection == .settings ? .primary : .secondary) + .padding(.horizontal, 12) + .padding(.vertical, 8) } + .buttonStyle(.plain) } .listStyle(.sidebar) .task { @@ -1379,7 +1379,6 @@ struct ContentCard: View { } struct SettingsContent: View { - @EnvironmentObject private var appState: AppState @StateObject private var vrmDownloader = VRMDownloader() @State private var modelsPath: String = "" @State private var libraryPath: String = "" @@ -1388,6 +1387,33 @@ struct SettingsContent: View { var body: some View { ScrollView { VStack(alignment: .leading, spacing: 24) { + // Send Feedback + Button { + if let url = URL(string: "mailto:info@arkavo.com") { + NSWorkspace.shared.open(url) + } + } label: { + HStack(spacing: 12) { + Image(systemName: "envelope.fill") + .font(.title2) + .foregroundStyle(Color.accentColor) + VStack(alignment: .leading, spacing: 2) { + Text("Send Feedback") + .font(.headline) + Text("Help us improve ArkavoCreator") + .font(.caption) + .foregroundStyle(.secondary) + } + Spacer() + Image(systemName: "arrow.up.right") + .foregroundStyle(.secondary) + } + .padding() + .background(Color(NSColor.controlBackgroundColor)) + .clipShape(RoundedRectangle(cornerRadius: 10)) + } + .buttonStyle(.plain) + // Library Path Section GroupBox { VStack(alignment: .leading, spacing: 12) { @@ -1475,22 +1501,6 @@ struct SettingsContent: View { AgentSettingsSection(agentService: agentService) } - // Feedback Toggle Section - VStack(alignment: .leading, spacing: 12) { - Text("Feedback") - .font(.headline) - - Toggle("Show Feedback Button", isOn: $appState.isFeedbackEnabled) - .toggleStyle(.switch) - - Text("When enabled, shows a feedback button in the toolbar for quick access to send feedback.") - .foregroundColor(.secondary) - .font(.callout) - } - .padding() - .background(Color(NSColor.controlBackgroundColor)) - .clipShape(RoundedRectangle(cornerRadius: 10)) - Spacer() } .padding() diff --git a/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift b/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift new file mode 100644 index 00000000..c06939c3 --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift @@ -0,0 +1,249 @@ +import MuseCore +import SwiftUI + +/// Compact side panel for the Publicist role — slides in from trailing edge on Dashboard +struct PublicistPanelView: View { + @Bindable var viewModel: PublicistViewModel + @Binding var isVisible: Bool + + var body: some View { + VStack(spacing: 0) { + // Header + HStack { + HStack(spacing: 8) { + Circle() + .fill(viewModel.modelManager.isReady ? Color.green : Color.gray) + .frame(width: 8, height: 8) + Text("Publicist") + .font(.headline) + } + + Spacer() + + if !viewModel.modelManager.isReady { + Button("Load") { + Task { await viewModel.modelManager.loadSelectedModel() } + } + .controlSize(.mini) + .buttonStyle(.bordered) + } + + Button { + withAnimation(.easeInOut(duration: 0.2)) { + isVisible = false + } + } label: { + Image(systemName: "xmark") + .font(.caption) + .foregroundStyle(.secondary) + } + .buttonStyle(.plain) + } + .padding(.horizontal, 16) + .padding(.vertical, 12) + + Divider() + + ScrollView { + VStack(alignment: .leading, spacing: 16) { + // Platform selector (compact) + platformSelector + + // Content type (compact) + contentTypeSelector + + // Source input + sourceInput + + // Generate + generateSection + + // Output + if !viewModel.generatedContent.isEmpty || viewModel.isGenerating { + outputSection + } + } + .padding(16) + } + } + .frame(width: 320) + .background(.ultraThinMaterial) + .overlay( + Rectangle().frame(width: 1).foregroundColor(.white.opacity(0.1)), + alignment: .leading + ) + } + + // MARK: - Platform Selector + + private var platformSelector: some View { + VStack(alignment: .leading, spacing: 6) { + Text("Platform") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 6) { + ForEach(PublicistPlatform.allCases, id: \.self) { platform in + let isSelected = viewModel.selectedPlatform == platform + Button { + viewModel.selectedPlatform = platform + } label: { + Text(platform.rawValue) + .font(.caption2.weight(isSelected ? .semibold : .regular)) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(isSelected ? AnyShapeStyle(Color.accentColor.opacity(0.2)) : AnyShapeStyle(.quaternary)) + .foregroundStyle(isSelected ? Color.accentColor : .primary) + .cornerRadius(6) + } + .buttonStyle(.plain) + } + } + } + } + } + + // MARK: - Content Type + + private var contentTypeSelector: some View { + VStack(alignment: .leading, spacing: 6) { + Text("Type") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + + HStack(spacing: 6) { + ForEach(PublicistContentType.allCases, id: \.self) { type in + let isSelected = viewModel.selectedContentType == type + Button { + viewModel.selectedContentType = type + } label: { + Text(type.rawValue) + .font(.caption2.weight(isSelected ? .semibold : .regular)) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(isSelected ? AnyShapeStyle(Color.accentColor.opacity(0.2)) : AnyShapeStyle(.quaternary)) + .foregroundStyle(isSelected ? Color.accentColor : .primary) + .cornerRadius(6) + } + .buttonStyle(.plain) + } + } + } + } + + // MARK: - Source Input + + private var sourceInput: some View { + VStack(alignment: .leading, spacing: 4) { + Text("Source") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + + TextEditor(text: $viewModel.sourceText) + .font(.caption) + .frame(minHeight: 40, maxHeight: 80) + .padding(6) + .background(.quaternary) + .cornerRadius(6) + } + } + + // MARK: - Generate + + private var generateSection: some View { + HStack { + if viewModel.isGenerating { + Button("Stop") { viewModel.stopGeneration() } + .controlSize(.small) + .buttonStyle(.bordered) + } else { + Button("Generate") { viewModel.generate() } + .controlSize(.small) + .buttonStyle(.borderedProminent) + .disabled(!viewModel.modelManager.isReady) + } + + Spacer() + + if let limit = viewModel.selectedPlatform.characterLimit { + Text("\(limit) max") + .font(.caption2) + .foregroundStyle(.tertiary) + } + } + } + + // MARK: - Output + + private var outputSection: some View { + VStack(alignment: .leading, spacing: 6) { + HStack { + Text("Output") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + Spacer() + if !viewModel.generatedContent.isEmpty { + HStack(spacing: 2) { + Text("\(viewModel.characterCount)") + .font(.caption2.monospacedDigit()) + .foregroundStyle(viewModel.isOverLimit ? .red : .secondary) + if let limit = viewModel.selectedPlatform.characterLimit { + Text("/ \(limit)") + .font(.caption2.monospacedDigit()) + .foregroundStyle(.tertiary) + } + } + } + } + + if viewModel.isGenerating { + VStack(alignment: .leading) { + Text(viewModel.streamingText) + .font(.caption) + .textSelection(.enabled) + ProgressView() + .controlSize(.mini) + } + .padding(8) + .frame(maxWidth: .infinity, alignment: .leading) + .background(.quaternary) + .cornerRadius(6) + } else { + Text(viewModel.generatedContent) + .font(.caption) + .textSelection(.enabled) + .padding(8) + .frame(maxWidth: .infinity, alignment: .leading) + .background(.quaternary) + .cornerRadius(6) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(viewModel.isOverLimit ? Color.red.opacity(0.5) : Color.clear, lineWidth: 1) + ) + } + + // Actions + if !viewModel.generatedContent.isEmpty && !viewModel.isGenerating { + HStack(spacing: 8) { + Button { viewModel.copyToClipboard() } label: { + Image(systemName: "doc.on.doc") + } + .help("Copy") + + Button { viewModel.generate() } label: { + Image(systemName: "arrow.clockwise") + } + .help("Regenerate") + + Button { viewModel.clearContent() } label: { + Image(systemName: "trash") + } + .help("Clear") + } + .buttonStyle(.bordered) + .controlSize(.mini) + } + } + } +} diff --git a/ArkavoCreator/ArkavoCreator/RecordView.swift b/ArkavoCreator/ArkavoCreator/RecordView.swift index ce67eacf..b6d8a991 100644 --- a/ArkavoCreator/ArkavoCreator/RecordView.swift +++ b/ArkavoCreator/ArkavoCreator/RecordView.swift @@ -100,6 +100,11 @@ struct RecordView: View { if producerViewModel == nil, let mm = modelManager { producerViewModel = ProducerViewModel(modelManager: mm) } + // Wire shared ModelManager and initialize Muse avatar for Sidekick + if museAvatarViewModel.modelManager == nil { + museAvatarViewModel.modelManager = modelManager + museAvatarViewModel.setup() + } syncViewModelState() if studioState.visualSource == .face { viewModel.bindPreviewStore(previewStore) From d4315950206fb50e07830f8c3998a2eedb777143 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Wed, 18 Mar 2026 21:57:07 -0400 Subject: [PATCH 04/14] Refine UI: fix visual hierarchy, consistency, and accessibility - Fix Settings sidebar using same NavigationLink highlight as other items - Change LIVE button from blue/purple gradient to red (broadcast standard) - Improve timer contrast and enlarge primary screen star badge - Format recording titles as human-readable dates instead of raw timestamps - Consolidate recording card metadata into single compact subtitle line - Demote Settings Reset button from destructive red to secondary gray - Fix Settings text hierarchy and compact the feedback banner - Add segmented control container styling to Publicist selectors - Add placeholder text and border to Publicist source TextEditor - Clarify character limit label from "max" to "chars" Co-Authored-By: Claude Opus 4.6 (1M context) --- ArkavoCreator/ArkavoCreator/ContentView.swift | 59 +++++++------------ .../Publicist/PublicistPanelView.swift | 36 ++++++++--- ArkavoCreator/ArkavoCreator/RecordView.swift | 10 ++-- ArkavoCreator/ArkavoCreator/Recording.swift | 16 +++-- .../ArkavoCreator/RecordingsLibraryView.swift | 15 ++--- 5 files changed, 71 insertions(+), 65 deletions(-) diff --git a/ArkavoCreator/ArkavoCreator/ContentView.swift b/ArkavoCreator/ArkavoCreator/ContentView.swift index 93f108bc..50198a28 100644 --- a/ArkavoCreator/ArkavoCreator/ContentView.swift +++ b/ArkavoCreator/ArkavoCreator/ContentView.swift @@ -1272,32 +1272,18 @@ struct Sidebar: View { } var body: some View { - VStack(spacing: 0) { - List(selection: $selectedSection) { - ForEach(availableSections.filter { $0 != .settings }, id: \.self) { section in - NavigationLink(value: section) { - Label(section.rawValue, systemImage: section.systemImage) - } + List(selection: $selectedSection) { + ForEach(availableSections.filter { $0 != .settings }, id: \.self) { section in + NavigationLink(value: section) { + Label(section.rawValue, systemImage: section.systemImage) } } - Divider() - - Button { - selectedSection = .settings - } label: { - HStack(spacing: 8) { - Image(systemName: selectedSection == .settings ? "gear.circle.fill" : "gear") - .font(.system(size: 14)) - Text("Settings") - .font(.subheadline) - Spacer() + Section { + NavigationLink(value: NavigationSection.settings) { + Label("Settings", systemImage: "gear") } - .foregroundStyle(selectedSection == .settings ? .primary : .secondary) - .padding(.horizontal, 12) - .padding(.vertical, 8) } - .buttonStyle(.plain) } .listStyle(.sidebar) .task { @@ -1393,24 +1379,21 @@ struct SettingsContent: View { NSWorkspace.shared.open(url) } } label: { - HStack(spacing: 12) { - Image(systemName: "envelope.fill") - .font(.title2) - .foregroundStyle(Color.accentColor) - VStack(alignment: .leading, spacing: 2) { - Text("Send Feedback") - .font(.headline) - Text("Help us improve ArkavoCreator") - .font(.caption) - .foregroundStyle(.secondary) - } + HStack(spacing: 8) { + Image(systemName: "envelope") + .font(.subheadline) + .foregroundStyle(.secondary) + Text("Send Feedback") + .font(.subheadline) Spacer() Image(systemName: "arrow.up.right") - .foregroundStyle(.secondary) + .font(.caption) + .foregroundStyle(.tertiary) } - .padding() + .padding(.vertical, 6) + .padding(.horizontal, 10) .background(Color(NSColor.controlBackgroundColor)) - .clipShape(RoundedRectangle(cornerRadius: 10)) + .clipShape(RoundedRectangle(cornerRadius: 8)) } .buttonStyle(.plain) @@ -1442,13 +1425,13 @@ struct SettingsContent: View { RecordingsFolderAccess.clearBookmark() updateLibraryPath() } - .foregroundColor(.red) + .foregroundColor(.secondary) } } Text("Select where recordings are saved. The app needs permission to write to this folder.") - .foregroundColor(.secondary) - .font(.callout) + .foregroundStyle(.tertiary) + .font(.caption) } } label: { Label("Library", systemImage: "folder") diff --git a/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift b/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift index c06939c3..b9b0cbaf 100644 --- a/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift +++ b/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift @@ -100,6 +100,9 @@ struct PublicistPanelView: View { .buttonStyle(.plain) } } + .padding(4) + .background(Color.black.opacity(0.2)) + .cornerRadius(8) } } } @@ -129,6 +132,9 @@ struct PublicistPanelView: View { .buttonStyle(.plain) } } + .padding(4) + .background(Color.black.opacity(0.2)) + .cornerRadius(8) } } @@ -140,12 +146,28 @@ struct PublicistPanelView: View { .font(.caption.weight(.semibold)) .foregroundStyle(.secondary) - TextEditor(text: $viewModel.sourceText) - .font(.caption) - .frame(minHeight: 40, maxHeight: 80) - .padding(6) - .background(.quaternary) - .cornerRadius(6) + ZStack(alignment: .topLeading) { + TextEditor(text: $viewModel.sourceText) + .font(.caption) + .frame(minHeight: 40, maxHeight: 80) + .padding(6) + .scrollContentBackground(.hidden) + + if viewModel.sourceText.isEmpty { + Text("Paste or type source content...") + .font(.caption) + .foregroundStyle(.tertiary) + .padding(.horizontal, 10) + .padding(.vertical, 14) + .allowsHitTesting(false) + } + } + .background(.quaternary) + .cornerRadius(6) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(Color.white.opacity(0.1)) + ) } } @@ -167,7 +189,7 @@ struct PublicistPanelView: View { Spacer() if let limit = viewModel.selectedPlatform.characterLimit { - Text("\(limit) max") + Text("\(limit) chars") .font(.caption2) .foregroundStyle(.tertiary) } diff --git a/ArkavoCreator/ArkavoCreator/RecordView.swift b/ArkavoCreator/ArkavoCreator/RecordView.swift index b6d8a991..e775ad5c 100644 --- a/ArkavoCreator/ArkavoCreator/RecordView.swift +++ b/ArkavoCreator/ArkavoCreator/RecordView.swift @@ -372,10 +372,10 @@ struct RecordView: View { if screen.isPrimary { Circle() .fill(Color.yellow) - .frame(width: 8, height: 8) + .frame(width: 12, height: 12) .overlay( Image(systemName: "star.fill") - .font(.system(size: 5)) + .font(.system(size: 8)) .foregroundColor(.black) ) .offset(x: 2, y: -2) @@ -410,13 +410,13 @@ struct RecordView: View { Text(activeDuration) .font(.system(size: 14, weight: .medium, design: .monospaced)) .monospacedDigit() - .foregroundStyle(isActive ? .primary : .secondary) + .foregroundStyle(.primary) } .frame(width: 90) .padding(.vertical, 6) .background(.ultraThinMaterial) .clipShape(Capsule()) - .opacity(isActive ? 1.0 : 0.5) + .opacity(isActive ? 1.0 : 0.7) // Producer Toggle if FeatureFlags.localAssistant { @@ -648,7 +648,7 @@ struct RecordView: View { .background( streamViewModel.isStreaming ? AnyShapeStyle(Color.red) - : AnyShapeStyle(LinearGradient(colors: [.blue, .purple], startPoint: .leading, endPoint: .trailing)) + : AnyShapeStyle(Color.red.opacity(0.4)) ) .foregroundColor(.white) .clipShape(Capsule()) diff --git a/ArkavoCreator/ArkavoCreator/Recording.swift b/ArkavoCreator/ArkavoCreator/Recording.swift index aeec06e6..a25d480a 100644 --- a/ArkavoCreator/ArkavoCreator/Recording.swift +++ b/ArkavoCreator/ArkavoCreator/Recording.swift @@ -113,6 +113,13 @@ struct Recording: Identifiable, Sendable { formatter.timeStyle = .short return formatter.string(from: date) } + + var formattedCardSubtitle: String { + let tf = DateFormatter() + tf.dateStyle = .none + tf.timeStyle = .short + return "\(tf.string(from: date)) \u{2022} \(formattedDuration) \u{2022} \(formattedFileSize)" + } } /// Manages recordings on disk @@ -197,10 +204,11 @@ final class RecordingsManager: ObservableObject { duration = 0 } - // Extract title from metadata or use filename - let title = url.deletingPathExtension().lastPathComponent - .replacingOccurrences(of: "arkavo_recording_", with: "") - .replacingOccurrences(of: "_", with: " ") + // Format title as a human-readable date + let titleFormatter = DateFormatter() + titleFormatter.dateStyle = .long + titleFormatter.timeStyle = .none + let title = titleFormatter.string(from: creationDate) let recording = Recording( id: UUID(), diff --git a/ArkavoCreator/ArkavoCreator/RecordingsLibraryView.swift b/ArkavoCreator/ArkavoCreator/RecordingsLibraryView.swift index b0ff6c98..ddb1e352 100644 --- a/ArkavoCreator/ArkavoCreator/RecordingsLibraryView.swift +++ b/ArkavoCreator/ArkavoCreator/RecordingsLibraryView.swift @@ -464,17 +464,10 @@ struct RecordingCard: View { .lineLimit(1) // Metadata - HStack(spacing: 12) { - Label(recording.formattedDate, systemImage: "calendar") - .font(.caption) - .foregroundColor(.secondary) - - Spacer() - - Label(recording.formattedFileSize, systemImage: "doc") - .font(.caption) - .foregroundColor(.secondary) - } + Text(recording.formattedCardSubtitle) + .font(.caption) + .foregroundColor(.secondary) + .lineLimit(1) } .padding(12) .background(Color(NSColor.controlBackgroundColor)) From 37058c62b20bcc5848ba81bba36e46da935f9775 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Wed, 18 Mar 2026 23:14:39 -0400 Subject: [PATCH 05/14] Overhaul UI: liquid glass materials, unified Producer panel, pro Studio layout Phase 2 visual overhaul: - Replace flat controlBackgroundColor with .ultraThinMaterial + specular gradient borders (top-left light source) on all cards - Add ambient gradient background behind NavigationSplitView for glass refraction - Spring animations replace all easeInOut transitions - Remove page headers (headerless pro layout), move Library metadata to native toolbar Studio restructure: - Fixed bottom control bar (edge-to-edge .regularMaterial, 68pt) - Consolidate Chat + Producer into unified Producer command center with integrated dense monospaced chat feed at bottom - Single panel toggle button replaces three separate panel buttons - Audio controls: mic/speaker toggle + chevron volume popovers - Scene picker moved next to timer as chevron popover - Control bar grouped: inputs (left), broadcast (center), toggle (right) - LIVE button pulses red shadow when streaming Recording cards: - Flush thumbnails clipped by outer card shape (no internal cornerRadius) - Adaptive grid columns (280-400pt) for fluid window resizing Co-Authored-By: Claude Opus 4.6 (1M context) --- ArkavoCreator/ArkavoCreator/ContentView.swift | 88 ++-- .../Producer/ProducerPanelView.swift | 87 +++- .../Publicist/PublicistPanelView.swift | 2 +- ArkavoCreator/ArkavoCreator/RecordView.swift | 453 ++++++++++-------- .../ArkavoCreator/RecordingsLibraryView.swift | 94 ++-- 5 files changed, 409 insertions(+), 315 deletions(-) diff --git a/ArkavoCreator/ArkavoCreator/ContentView.swift b/ArkavoCreator/ArkavoCreator/ContentView.swift index 50198a28..3da8bcaf 100644 --- a/ArkavoCreator/ArkavoCreator/ContentView.swift +++ b/ArkavoCreator/ArkavoCreator/ContentView.swift @@ -21,28 +21,35 @@ struct ContentView: View { ) var body: some View { - NavigationSplitView { - Sidebar( - selectedSection: $selectedSection, - patreonClient: patreonClient, - redditClient: redditClient, - blueskyClient: blueskyClient, - youtubeClient: youtubeClient - ) - } detail: { - SectionContainer( - selectedSection: selectedSection, - patreonClient: patreonClient, - redditClient: redditClient, - micropubClient: micropubClient, - blueskyClient: blueskyClient, - youtubeClient: youtubeClient, - twitchClient: twitchClient, - agentService: agentService, - modelManager: modelManager + ZStack { + LinearGradient( + colors: [Color(white: 0.1), Color(white: 0.15), Color(white: 0.08)], + startPoint: .topLeading, + endPoint: .bottomTrailing ) - .navigationTitle(selectedSection.rawValue) - .navigationSubtitle(selectedSection.subtitle) + .ignoresSafeArea() + + NavigationSplitView { + Sidebar( + selectedSection: $selectedSection, + patreonClient: patreonClient, + redditClient: redditClient, + blueskyClient: blueskyClient, + youtubeClient: youtubeClient + ) + } detail: { + SectionContainer( + selectedSection: selectedSection, + patreonClient: patreonClient, + redditClient: redditClient, + micropubClient: micropubClient, + blueskyClient: blueskyClient, + youtubeClient: youtubeClient, + twitchClient: twitchClient, + agentService: agentService, + modelManager: modelManager + ) + } } .environmentObject(appState) .onChange(of: selectedSection) { _, newValue in @@ -254,9 +261,10 @@ struct SectionContainer: View { ForEach(twitchClient.channelTags.prefix(5), id: \.self) { tag in Text(tag) .font(.caption2) + .foregroundStyle(.secondary) .padding(.horizontal, 6) .padding(.vertical, 3) - .background(Color.purple.opacity(0.15)) + .background(Color.white.opacity(0.08)) .clipShape(Capsule()) } } @@ -407,7 +415,7 @@ struct SectionContainer: View { } } .onHover { hovering in - withAnimation(.easeInOut(duration: 0.15)) { + withAnimation(.spring(response: 0.3, dampingFraction: 0.75)) { twitchCardHovered = hovering } } @@ -853,7 +861,7 @@ struct SectionContainer: View { if publicistViewModel == nil { publicistViewModel = PublicistViewModel(modelManager: modelManager) } - withAnimation(.easeInOut(duration: 0.2)) { + withAnimation(.spring(response: 0.3, dampingFraction: 0.75)) { showPublicistPanel.toggle() } } label: { @@ -897,7 +905,7 @@ struct SectionContainer: View { } } } - .animation(.smooth, value: selectedSection) + .animation(.spring(response: 0.35, dampingFraction: 0.85), value: selectedSection) } } @@ -983,8 +991,20 @@ struct DashboardCard: View { } .frame(maxWidth: .infinity, alignment: .leading) .padding() - .background(Color(NSColor.controlBackgroundColor)) + .background(.ultraThinMaterial) .clipShape(RoundedRectangle(cornerRadius: 10)) + .overlay( + RoundedRectangle(cornerRadius: 10) + .strokeBorder( + LinearGradient( + colors: [.white.opacity(0.25), .white.opacity(0.02)], + startPoint: .topLeading, + endPoint: .bottomTrailing + ), + lineWidth: 0.5 + ) + ) + .shadow(color: .black.opacity(0.3), radius: 12, x: 0, y: 6) } } @@ -1087,7 +1107,7 @@ struct PreviewAlert: View { .buttonStyle(.borderedProminent) } .padding() - .background(.background.secondary) + .background(.ultraThinMaterial) .clipShape(RoundedRectangle(cornerRadius: 10)) } } @@ -1117,7 +1137,7 @@ struct FeatureCard: View { } .padding() .frame(height: 160) - .background(.background.secondary) + .background(.ultraThinMaterial) .clipShape(RoundedRectangle(cornerRadius: 10)) } } @@ -1207,13 +1227,13 @@ struct IconRail: View { hoverTask = Task { try? await Task.sleep(for: .milliseconds(300)) guard !Task.isCancelled else { return } - withAnimation(.easeInOut(duration: 0.2)) { + withAnimation(.spring(response: 0.3, dampingFraction: 0.75)) { isExpanded = true } } } else { // Collapse immediately when leaving - withAnimation(.easeInOut(duration: 0.2)) { + withAnimation(.spring(response: 0.3, dampingFraction: 0.75)) { isExpanded = false } } @@ -1354,7 +1374,7 @@ struct ContentCard: View { .controlSize(.small) } .padding(12) - .background(Color(nsColor: .controlBackgroundColor)) + .background(.ultraThinMaterial) .clipShape(RoundedRectangle(cornerRadius: 8)) } } @@ -1392,7 +1412,7 @@ struct SettingsContent: View { } .padding(.vertical, 6) .padding(.horizontal, 10) - .background(Color(NSColor.controlBackgroundColor)) + .background(.ultraThinMaterial) .clipShape(RoundedRectangle(cornerRadius: 8)) } .buttonStyle(.plain) @@ -1475,7 +1495,7 @@ struct SettingsContent: View { .font(.callout) } .padding() - .background(Color(NSColor.controlBackgroundColor)) + .background(.ultraThinMaterial) .clipShape(RoundedRectangle(cornerRadius: 10)) } @@ -1552,7 +1572,7 @@ struct AgentSettingsSection: View { .font(.callout) } .padding() - .background(Color(NSColor.controlBackgroundColor)) + .background(.ultraThinMaterial) .clipShape(RoundedRectangle(cornerRadius: 10)) } } diff --git a/ArkavoCreator/ArkavoCreator/Producer/ProducerPanelView.swift b/ArkavoCreator/ArkavoCreator/Producer/ProducerPanelView.swift index ce5e742a..f620bfff 100644 --- a/ArkavoCreator/ArkavoCreator/Producer/ProducerPanelView.swift +++ b/ArkavoCreator/ArkavoCreator/Producer/ProducerPanelView.swift @@ -1,10 +1,11 @@ import MuseCore import SwiftUI -/// Private overlay panel for the Producer role — slides in from trailing edge in Studio +/// Unified Producer panel — command center with stream health, actions, suggestions, and chat feed struct ProducerPanelView: View { var viewModel: ProducerViewModel @Binding var isVisible: Bool + var chatViewModel: ChatPanelViewModel? var body: some View { VStack(spacing: 0) { @@ -12,7 +13,7 @@ struct ProducerPanelView: View { HStack { HStack(spacing: 8) { Circle() - .fill(viewModel.streamState.isLive ? Color.green : Color.gray) + .fill(viewModel.modelManager.isReady ? Color.green : Color.gray) .frame(width: 8, height: 8) Text("Producer") .font(.headline) @@ -21,7 +22,7 @@ struct ProducerPanelView: View { Spacer() Button { - withAnimation(.easeInOut(duration: 0.2)) { + withAnimation(.spring(response: 0.3, dampingFraction: 0.75)) { isVisible = false } } label: { @@ -36,30 +37,84 @@ struct ProducerPanelView: View { Divider() + // Sections 1-3: Health, Actions, Suggestions (fixed height, scrollable) ScrollView { VStack(alignment: .leading, spacing: 16) { - // Stream Health streamHealthSection - Divider() - - // Quick Actions quickActionsSection - Divider() - - // Suggestions suggestionsSection } .padding(16) } + .frame(maxHeight: 320) + + Divider() + + // Section 4: Chat Monitor (fills remaining space) + if let chatVM = chatViewModel { + chatMonitorSection(chatVM) + } else { + VStack(spacing: 8) { + Image(systemName: "bubble.left.and.bubble.right") + .font(.title3) + .foregroundStyle(.tertiary) + Text("Chat appears when streaming") + .font(.caption) + .foregroundStyle(.tertiary) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + } + } + + // MARK: - Chat Monitor + + @ViewBuilder + private func chatMonitorSection(_ chatVM: ChatPanelViewModel) -> some View { + VStack(spacing: 0) { + // Chat header + HStack { + Circle() + .fill(chatVM.isConnected ? Color.green : Color.gray) + .frame(width: 6, height: 6) + Text("Chat") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + Spacer() + Text("\(chatVM.messages.count)") + .font(.caption2.monospacedDigit()) + .foregroundStyle(.tertiary) + } + .padding(.horizontal, 12) + .padding(.vertical, 6) + + // Dense chat feed + ScrollViewReader { proxy in + ScrollView { + LazyVStack(alignment: .leading, spacing: 2) { + ForEach(chatVM.messages) { message in + Text("\(Text(message.displayName).bold()): \(message.content)") + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.primary.opacity(0.85)) + .lineLimit(2) + .padding(.horizontal, 8) + .padding(.vertical, 1) + .id(message.id) + } + } + } + .onChange(of: chatVM.messages.count) { _, _ in + if let lastID = chatVM.messages.last?.id { + withAnimation(.easeOut(duration: 0.1)) { + proxy.scrollTo(lastID, anchor: .bottom) + } + } + } + } } - .frame(width: 300) - .background(.ultraThinMaterial) - .overlay( - Rectangle().frame(width: 1).foregroundColor(.white.opacity(0.1)), - alignment: .leading - ) + .frame(maxHeight: .infinity) } // MARK: - Stream Health diff --git a/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift b/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift index b9b0cbaf..83581da6 100644 --- a/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift +++ b/ArkavoCreator/ArkavoCreator/Publicist/PublicistPanelView.swift @@ -29,7 +29,7 @@ struct PublicistPanelView: View { } Button { - withAnimation(.easeInOut(duration: 0.2)) { + withAnimation(.spring(response: 0.3, dampingFraction: 0.75)) { isVisible = false } } label: { diff --git a/ArkavoCreator/ArkavoCreator/RecordView.swift b/ArkavoCreator/ArkavoCreator/RecordView.swift index e775ad5c..10423cac 100644 --- a/ArkavoCreator/ArkavoCreator/RecordView.swift +++ b/ArkavoCreator/ArkavoCreator/RecordView.swift @@ -19,9 +19,7 @@ struct RecordView: View { @StateObject private var museAvatarViewModel = MuseAvatarViewModel() @State private var enableScreen: Bool = false @State private var showStreamSetup: Bool = false - @State private var showInspector: Bool = false - @State private var showChat: Bool = false - @State private var showProducerPanel: Bool = false + @State private var showRightPanel: Bool = false @State private var chatViewModel = ChatPanelViewModel() @State private var producerViewModel: ProducerViewModel? @State private var pulsing: Bool = false @@ -30,6 +28,10 @@ struct RecordView: View { // Scene state restoration @State private var preSceneMicEnabled: Bool = true @State private var preSceneVisualSource: VisualSource? = .face + @State private var isLivePulsing: Bool = false + @State private var showMicPopover: Bool = false + @State private var showAudioPopover: Bool = false + @State private var showScenePopover: Bool = false // Shared state (not part of init) @ObservedObject private var previewStore = CameraPreviewStore.shared @@ -42,14 +44,9 @@ struct RecordView: View { .fill(.white.opacity(0.1)) .frame(height: 1) - // MARK: - Main Stage + Inspector + // MARK: - Main Stage + Panels HStack(spacing: 0) { - // Chat Panel (left side) - if showChat { - ChatPanelView(viewModel: chatViewModel, isVisible: $showChat) - .transition(.move(edge: .leading)) - } - + // Stage ZStack { // Ambient Background LinearGradient( @@ -57,7 +54,6 @@ struct RecordView: View { startPoint: .topLeading, endPoint: .bottomTrailing ) - .ignoresSafeArea() stageCompositionView .clipped() @@ -67,33 +63,38 @@ struct RecordView: View { SceneOverlayView(scene: studioState.activeScene) } } + .clipped() .frame(maxWidth: .infinity, maxHeight: .infinity) - if let producerVM = producerViewModel, showProducerPanel { - ProducerPanelView(viewModel: producerVM, isVisible: $showProducerPanel) - .transition(.move(edge: .trailing)) - } - - if showInspector { - InspectorPanel( - visualSource: studioState.visualSource, - recordViewModel: viewModel, - avatarViewModel: avatarViewModel, - isVisible: $showInspector, - onLoadAvatarModel: { - Task { await avatarViewModel.loadSelectedModel() } - } + // Producer Panel (unified command center) + if showRightPanel, let producerVM = producerViewModel { + ProducerPanelView( + viewModel: producerVM, + isVisible: $showRightPanel, + chatViewModel: chatViewModel ) + .frame(width: 300) + .background(.ultraThinMaterial) + .overlay(alignment: .leading) { + Rectangle().frame(width: 1).foregroundStyle(.white.opacity(0.1)) + } .transition(.move(edge: .trailing)) } + } + .frame(maxHeight: .infinity) - // MARK: - Bottom Control Bar + // MARK: - Fixed Bottom Control Panel studioControlBar - .padding(.horizontal, 16) - .padding(.vertical, 12) - .background(.ultraThinMaterial) - .overlay(Rectangle().frame(height: 1).foregroundColor(.white.opacity(0.1)), alignment: .top) + .padding(.horizontal, 24) + .frame(height: 68) + .frame(maxWidth: .infinity) + .background(.regularMaterial) + .overlay(alignment: .top) { + Rectangle() + .frame(height: 1) + .foregroundStyle(.white.opacity(0.1)) + } } .navigationTitle("Studio") .onAppear { @@ -316,93 +317,87 @@ struct RecordView: View { // MARK: - Studio Control Bar private var studioControlBar: some View { - HStack(spacing: 16) { - // Left: Visual Source Toggle (Face/Avatar - both can be off for audio-only) - HStack(spacing: 4) { - ForEach(VisualSource.availableSources) { source in - let isSelected = studioState.visualSource == source - Button { - studioState.toggleVisualSource(source) - } label: { - Image(systemName: source.icon) - .font(.system(size: 14)) - .frame(width: 32, height: 32) - .background(isSelected ? Color.accentColor.opacity(0.3) : Color.clear) - .background(.regularMaterial) - .cornerRadius(6) - .overlay( - RoundedRectangle(cornerRadius: 6) - .stroke(isSelected ? Color.accentColor : Color.clear, lineWidth: 1) - ) - } - .buttonStyle(.plain) - .help(isSelected ? "Disable \(source.rawValue)" : "Enable \(source.rawValue)") - .accessibilityIdentifier("Source_\(source.rawValue)") - } - } - - // Screen Selection - HStack(spacing: 4) { - ForEach(viewModel.availableScreens) { screen in - let isSelected = enableScreen && viewModel.selectedScreenID == screen.displayID - Button { - if isSelected { - // Deselect (turn off screen share) - enableScreen = false - } else { - // Select this screen - viewModel.selectScreen(screen) - enableScreen = true - } - } label: { - ZStack(alignment: .topTrailing) { - Image(systemName: isSelected ? "rectangle.inset.filled.on.rectangle" : "desktopcomputer") + HStack { + // LEFT: Inputs & Sources + HStack(spacing: 8) { + // Visual source toggles + HStack(spacing: 4) { + ForEach(VisualSource.availableSources) { source in + let isSelected = studioState.visualSource == source + Button { + studioState.toggleVisualSource(source) + } label: { + Image(systemName: source.icon) .font(.system(size: 14)) - .foregroundStyle(isSelected ? Color.accentColor : .primary) - .padding(8) + .frame(width: 32, height: 32) .background(isSelected ? Color.accentColor.opacity(0.3) : Color.clear) .background(.regularMaterial) - .cornerRadius(8) + .cornerRadius(6) .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(isSelected ? Color.accentColor : Color.clear, lineWidth: 2) + RoundedRectangle(cornerRadius: 6) + .stroke(isSelected ? Color.accentColor : Color.clear, lineWidth: 1) ) + } + .buttonStyle(.plain) + .help(isSelected ? "Disable \(source.rawValue)" : "Enable \(source.rawValue)") + .accessibilityIdentifier("Source_\(source.rawValue)") + } + } - // Primary screen indicator (star badge) - if screen.isPrimary { - Circle() - .fill(Color.yellow) - .frame(width: 12, height: 12) + // Screen selection + HStack(spacing: 4) { + ForEach(viewModel.availableScreens) { screen in + let isSelected = enableScreen && viewModel.selectedScreenID == screen.displayID + Button { + if isSelected { + enableScreen = false + } else { + viewModel.selectScreen(screen) + enableScreen = true + } + } label: { + ZStack(alignment: .topTrailing) { + Image(systemName: isSelected ? "rectangle.inset.filled.on.rectangle" : "desktopcomputer") + .font(.system(size: 14)) + .foregroundStyle(isSelected ? Color.accentColor : .primary) + .padding(8) + .background(isSelected ? Color.accentColor.opacity(0.3) : Color.clear) + .background(.regularMaterial) + .cornerRadius(8) .overlay( - Image(systemName: "star.fill") - .font(.system(size: 8)) - .foregroundColor(.black) + RoundedRectangle(cornerRadius: 8) + .stroke(isSelected ? Color.accentColor : Color.clear, lineWidth: 2) ) - .offset(x: 2, y: -2) + + if screen.isPrimary { + Circle() + .fill(Color.yellow) + .frame(width: 12, height: 12) + .overlay( + Image(systemName: "star.fill") + .font(.system(size: 8)) + .foregroundColor(.black) + ) + .offset(x: 2, y: -2) + } } } + .buttonStyle(.plain) + .accessibilityIdentifier("Screen_\(screen.id)") + .help(screen.isPrimary ? "\(screen.name) (Primary)" : screen.name) } - .buttonStyle(.plain) - .accessibilityIdentifier("Screen_\(screen.id)") - .help(screen.isPrimary ? "\(screen.name) (Primary)" : screen.name) } - } - audioAndSceneControls - - Spacer() + audioAndSceneControls + } + .frame(maxWidth: .infinity, alignment: .leading) - // Center: Dual Action Buttons (REC + LIVE) + // CENTER: Broadcasting HStack(spacing: 12) { recordingActionButton streamingActionButton - } - Spacer() - - // Right: Recording Duration + Settings - HStack(spacing: 12) { - // Duration (recording or streaming) + // Duration HStack(spacing: 6) { Circle() .fill(viewModel.isRecording ? .red : (streamViewModel.isStreaming ? .purple : .clear)) @@ -418,137 +413,162 @@ struct RecordView: View { .clipShape(Capsule()) .opacity(isActive ? 1.0 : 0.7) - // Producer Toggle - if FeatureFlags.localAssistant { - Button { - withAnimation(.easeInOut(duration: 0.2)) { - showProducerPanel.toggle() - } - } label: { - Image(systemName: "theatermask.and.paintbrush") + // Scene picker + Button { showScenePopover.toggle() } label: { + HStack(spacing: 4) { + Image(systemName: studioState.activeScene.icon) .font(.system(size: 14)) - .padding(8) - .foregroundStyle(showProducerPanel ? .primary : .secondary) - .background(showProducerPanel ? Color.accentColor.opacity(0.2) : Color.clear) - .background(.regularMaterial) - .cornerRadius(8) - } - .buttonStyle(.plain) - .help("Toggle Producer Panel (⌘P)") - .keyboardShortcut("p", modifiers: .command) - .accessibilityIdentifier("Toggle_Producer") - } - - // Chat Toggle (Twitch only) - if streamViewModel.selectedPlatform == .twitch { - Button { - withAnimation(.easeInOut(duration: 0.2)) { - showChat.toggle() - } - } label: { - Image(systemName: "bubble.left.and.bubble.right") - .font(.system(size: 14)) - .padding(8) - .foregroundStyle(showChat ? .primary : .secondary) - .background(showChat ? Color.accentColor.opacity(0.2) : Color.clear) - .background(.regularMaterial) - .cornerRadius(8) - } - .buttonStyle(.plain) - .help("Toggle Chat Panel") - } - - // Inspector Toggle - Button { - withAnimation(.easeInOut(duration: 0.2)) { - showInspector.toggle() + Image(systemName: "chevron.up") + .font(.system(size: 8, weight: .bold)) } - } label: { - Image(systemName: "slider.horizontal.3") - .font(.system(size: 14)) - .padding(8) - .foregroundStyle(showInspector ? .primary : .secondary) - .background(showInspector ? Color.accentColor.opacity(0.2) : Color.clear) - .background(.regularMaterial) - .cornerRadius(8) - } - .buttonStyle(.plain) - .help("Toggle Inspector (⌘I)") - .keyboardShortcut("i", modifiers: .command) - } - } - } - - private var audioAndSceneControls: some View { - HStack(spacing: 8) { - // Mic Toggle - Button { - viewModel.enableMicrophone.toggle() - } label: { - Image(systemName: viewModel.enableMicrophone ? "mic.fill" : "mic.slash") - .font(.system(size: 14)) .padding(8) - .background(viewModel.enableMicrophone ? Color.accentColor.opacity(0.2) : Color.clear) + .background(studioState.isSceneOverlayActive ? Color.orange.opacity(0.3) : Color.clear) .background(.regularMaterial) .cornerRadius(8) .overlay( RoundedRectangle(cornerRadius: 8) - .stroke(viewModel.enableMicrophone ? Color.accentColor : Color.clear, lineWidth: 1) + .stroke(studioState.isSceneOverlayActive ? Color.orange : Color.clear, lineWidth: 1) ) + } + .buttonStyle(.plain) + .help("Scene Presets") + .popover(isPresented: $showScenePopover, arrowEdge: .top) { + scenePopoverContent + } } - .buttonStyle(.plain) - .accessibilityIdentifier("Toggle_Mic") - .help("Toggle Microphone") - // Desktop Audio Toggle + // RIGHT: Panel Toggle (single button) Button { - viewModel.toggleDesktopAudio() + withAnimation(.spring(response: 0.3, dampingFraction: 0.75)) { + showRightPanel.toggle() + } } label: { - Image(systemName: viewModel.enableDesktopAudio ? "speaker.wave.2.fill" : "speaker.slash") + Image(systemName: "slider.horizontal.3") .font(.system(size: 14)) .padding(8) - .background(viewModel.enableDesktopAudio ? Color.accentColor.opacity(0.2) : Color.clear) + .foregroundStyle(showRightPanel ? .primary : .secondary) + .background(showRightPanel ? Color.accentColor.opacity(0.2) : Color.clear) .background(.regularMaterial) .cornerRadius(8) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(viewModel.enableDesktopAudio ? Color.accentColor : Color.clear, lineWidth: 1) - ) } .buttonStyle(.plain) - .accessibilityIdentifier("Toggle_DesktopAudio") - .help("Toggle Desktop Audio") - - // Scene Picker - Menu { - ForEach(ScenePreset.allCases) { scene in - Button { - switchScene(to: scene) - } label: { - Label(scene.rawValue, systemImage: scene.icon) + .help("Toggle Panel (⌘P)") + .keyboardShortcut("p", modifiers: .command) + .frame(maxWidth: .infinity, alignment: .trailing) + } + } + + private var scenePopoverContent: some View { + VStack(alignment: .leading, spacing: 4) { + Text("Scene") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + .padding(.horizontal, 4) + ForEach(ScenePreset.allCases, id: \.self) { scene in + Button { + switchScene(to: scene) + showScenePopover = false + } label: { + Label(scene.rawValue, systemImage: scene.icon) + .font(.subheadline) + .padding(.vertical, 4) + .padding(.horizontal, 8) + .frame(maxWidth: .infinity, alignment: .leading) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + } + } + .padding(8) + .frame(width: 180) + } + + private var audioAndSceneControls: some View { + HStack(spacing: 8) { + // Mic Toggle + Volume Popover + HStack(spacing: 2) { + Button { + viewModel.enableMicrophone.toggle() + } label: { + Image(systemName: viewModel.enableMicrophone ? "mic.fill" : "mic.slash") + .font(.system(size: 14)) + .padding(8) + .background(viewModel.enableMicrophone ? Color.accentColor.opacity(0.2) : Color.clear) + .background(.regularMaterial) + .clipShape(UnevenRoundedRectangle(topLeadingRadius: 8, bottomLeadingRadius: 8, bottomTrailingRadius: 0, topTrailingRadius: 0)) + } + .buttonStyle(.plain) + .accessibilityIdentifier("Toggle_Mic") + .help("Toggle Microphone") + + Button { showMicPopover.toggle() } label: { + Image(systemName: "chevron.up") + .font(.system(size: 8, weight: .bold)) + .frame(width: 16, height: 32) + .background(.regularMaterial) + .clipShape(UnevenRoundedRectangle(topLeadingRadius: 0, bottomLeadingRadius: 0, bottomTrailingRadius: 8, topTrailingRadius: 8)) + } + .buttonStyle(.plain) + .popover(isPresented: $showMicPopover, arrowEdge: .top) { + VStack(spacing: 8) { + Text("Microphone") + .font(.caption.weight(.semibold)) + Slider(value: $viewModel.micVolume, in: 0...1) + .frame(width: 140) + Text("\(Int(viewModel.micVolume * 100))%") + .font(.caption2) + .foregroundStyle(.secondary) } + .padding(12) } - } label: { - HStack(spacing: 4) { - Image(systemName: studioState.activeScene.icon) + } + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(viewModel.enableMicrophone ? Color.accentColor : Color.clear, lineWidth: 1) + ) + + // Desktop Audio Toggle + Volume Popover + HStack(spacing: 2) { + Button { + viewModel.toggleDesktopAudio() + } label: { + Image(systemName: viewModel.enableDesktopAudio ? "speaker.wave.2.fill" : "speaker.slash") .font(.system(size: 14)) - if studioState.isSceneOverlayActive { - Text(studioState.activeScene.rawValue) - .font(.caption.weight(.medium)) + .padding(8) + .background(viewModel.enableDesktopAudio ? Color.accentColor.opacity(0.2) : Color.clear) + .background(.regularMaterial) + .clipShape(UnevenRoundedRectangle(topLeadingRadius: 8, bottomLeadingRadius: 8, bottomTrailingRadius: 0, topTrailingRadius: 0)) + } + .buttonStyle(.plain) + .accessibilityIdentifier("Toggle_DesktopAudio") + .help("Toggle Desktop Audio") + + Button { showAudioPopover.toggle() } label: { + Image(systemName: "chevron.up") + .font(.system(size: 8, weight: .bold)) + .frame(width: 16, height: 32) + .background(.regularMaterial) + .clipShape(UnevenRoundedRectangle(topLeadingRadius: 0, bottomLeadingRadius: 0, bottomTrailingRadius: 8, topTrailingRadius: 8)) + } + .buttonStyle(.plain) + .popover(isPresented: $showAudioPopover, arrowEdge: .top) { + VStack(spacing: 8) { + Text("Desktop Audio") + .font(.caption.weight(.semibold)) + Slider(value: $viewModel.desktopAudioVolume, in: 0...1) + .frame(width: 140) + Text("\(Int(viewModel.desktopAudioVolume * 100))%") + .font(.caption2) + .foregroundStyle(.secondary) } + .padding(12) } - .padding(8) - .background(studioState.isSceneOverlayActive ? Color.orange.opacity(0.3) : Color.clear) - .background(.regularMaterial) - .cornerRadius(8) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(studioState.isSceneOverlayActive ? Color.orange : Color.clear, lineWidth: 1) - ) } - .menuStyle(.borderlessButton) - .fixedSize() - .help("Scene Presets") + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(viewModel.enableDesktopAudio ? Color.accentColor : Color.clear, lineWidth: 1) + ) + } } @@ -635,7 +655,6 @@ struct RecordView: View { } } label: { HStack(spacing: 6) { - // Live indicator dot (always present for consistent width) Circle() .fill(streamViewModel.isStreaming ? .white : .clear) .frame(width: 8, height: 8) @@ -648,15 +667,25 @@ struct RecordView: View { .background( streamViewModel.isStreaming ? AnyShapeStyle(Color.red) - : AnyShapeStyle(Color.red.opacity(0.4)) + : AnyShapeStyle(.regularMaterial) ) - .foregroundColor(.white) + .foregroundColor(streamViewModel.isStreaming ? .white : .primary) .clipShape(Capsule()) } .buttonStyle(.plain) + .shadow(color: Color.red.opacity(isLivePulsing ? 0.5 : 0.0), radius: isLivePulsing ? 8 : 0) .disabled(streamViewModel.isConnecting) .accessibilityIdentifier("Btn_GoLive") .frame(width: 120) + .onChange(of: streamViewModel.isStreaming) { _, isStreaming in + if isStreaming { + withAnimation(.easeInOut(duration: 1.0).repeatForever(autoreverses: true)) { + isLivePulsing = true + } + } else { + withAnimation { isLivePulsing = false } + } + } } // MARK: - Helpers @@ -716,7 +745,7 @@ struct RecordView: View { preSceneVisualSource = studioState.visualSource } - withAnimation(.easeInOut(duration: 0.3)) { + withAnimation(.spring(response: 0.3, dampingFraction: 0.75)) { studioState.activeScene = scene } @@ -807,7 +836,7 @@ struct RecordView: View { // Auto-connect Twitch chat if streamViewModel.selectedPlatform == .twitch { chatViewModel.connect(twitchClient: twitchClient) - withAnimation { showChat = true } + withAnimation { showRightPanel = true } } } catch { streamViewModel.error = error.localizedDescription @@ -816,7 +845,7 @@ struct RecordView: View { private func stopStreaming() async { chatViewModel.disconnect() - showChat = false + showRightPanel = false await streamViewModel.stopStreaming() } } diff --git a/ArkavoCreator/ArkavoCreator/RecordingsLibraryView.swift b/ArkavoCreator/ArkavoCreator/RecordingsLibraryView.swift index ddb1e352..34baa95b 100644 --- a/ArkavoCreator/ArkavoCreator/RecordingsLibraryView.swift +++ b/ArkavoCreator/ArkavoCreator/RecordingsLibraryView.swift @@ -16,7 +16,7 @@ struct RecordingsLibraryView: View { @State private var protectionError: String? @State private var showingProtectionError = false @State private var recordingToDelete: Recording? - @State private var gridColumns = [GridItem(.adaptive(minimum: 200, maximum: 300), spacing: 16)] + @State private var gridColumns = [GridItem(.adaptive(minimum: 280, maximum: 400), spacing: 20)] // Iroh publishing state @State private var isPublishing = false @@ -29,17 +29,11 @@ struct RecordingsLibraryView: View { var body: some View { VStack(spacing: 0) { - // Header - headerView - - Divider() - - // Content if manager.recordings.isEmpty { emptyStateView } else { ScrollView { - LazyVGrid(columns: gridColumns, spacing: 16) { + LazyVGrid(columns: gridColumns, spacing: 20) { ForEach(manager.recordings) { recording in RecordingCard(recording: recording) .accessibilityIdentifier("RecordingCard_\(recording.id)") @@ -55,6 +49,16 @@ struct RecordingsLibraryView: View { } } } + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button { + Task { await manager.loadRecordings() } + } label: { + Image(systemName: "arrow.clockwise") + } + .help("Refresh") + } + } .sheet(item: $playerRecording) { recording in VideoPlayerView(recording: recording) } @@ -137,36 +141,13 @@ struct RecordingsLibraryView: View { // MARK: - View Components - private var headerView: some View { - HStack { - Text("Recordings") - .font(.title2) - .fontWeight(.semibold) - - Spacer() - - Text("\(manager.recordings.count) recording\(manager.recordings.count == 1 ? "" : "s")") - .foregroundColor(.secondary) - .font(.subheadline) - - Button { - Task { - await manager.loadRecordings() - } - } label: { - Image(systemName: "arrow.clockwise") - } - .help("Refresh") - } - .padding() - } - private var emptyStateView: some View { VStack(spacing: 16) { if manager.needsFolderSelection { Image(systemName: "folder.badge.questionmark") .font(.system(size: 60)) .foregroundColor(.blue) + .shadow(color: .blue.opacity(0.15), radius: 20) Text("Choose Recordings Folder") .font(.title2) @@ -185,6 +166,7 @@ struct RecordingsLibraryView: View { Image(systemName: "video.slash") .font(.system(size: 60)) .foregroundColor(.secondary) + .shadow(color: .blue.opacity(0.15), radius: 20) Text("No Recordings Yet") .font(.title2) @@ -379,8 +361,8 @@ struct RecordingCard: View { @State private var tdfStatus: Recording.TDFProtectionStatus? var body: some View { - VStack(alignment: .leading, spacing: 8) { - // Thumbnail + VStack(alignment: .leading, spacing: 0) { + // Thumbnail (flush to card edges — outer clipShape handles corners) ZStack { if let thumbnail = thumbnail { Image(nsImage: thumbnail) @@ -388,12 +370,10 @@ struct RecordingCard: View { .aspectRatio(contentMode: .fill) .frame(height: 150) .clipped() - .cornerRadius(8) } else { Rectangle() .fill(Color.secondary.opacity(0.2)) .frame(height: 150) - .cornerRadius(8) .overlay { ProgressView() } @@ -402,7 +382,6 @@ struct RecordingCard: View { // Badges (top-left) VStack { HStack(spacing: 4) { - // TDF Badge if let status = tdfStatus, status.isProtected { HStack(spacing: 4) { Image(systemName: "lock.shield.fill") @@ -419,7 +398,6 @@ struct RecordingCard: View { .accessibilityIdentifier("TDF3Badge") } - // C2PA Badge if let status = c2paStatus, status.isSigned { HStack(spacing: 4) { Image(systemName: status.isValid ? "checkmark.seal.fill" : "exclamationmark.triangle.fill") @@ -458,21 +436,33 @@ struct RecordingCard: View { } } - // Title - Text(recording.title) - .font(.headline) - .lineLimit(1) + // Title + Metadata (with padding) + VStack(alignment: .leading, spacing: 4) { + Text(recording.title) + .font(.headline) + .lineLimit(1) - // Metadata - Text(recording.formattedCardSubtitle) - .font(.caption) - .foregroundColor(.secondary) - .lineLimit(1) - } - .padding(12) - .background(Color(NSColor.controlBackgroundColor)) - .cornerRadius(12) - .shadow(color: .black.opacity(0.1), radius: 4, x: 0, y: 2) + Text(recording.formattedCardSubtitle) + .font(.caption) + .foregroundColor(.secondary) + .lineLimit(1) + } + .padding(12) + } + .background(.ultraThinMaterial) + .clipShape(RoundedRectangle(cornerRadius: 12)) + .overlay( + RoundedRectangle(cornerRadius: 12) + .strokeBorder( + LinearGradient( + colors: [.white.opacity(0.25), .white.opacity(0.02)], + startPoint: .topLeading, + endPoint: .bottomTrailing + ), + lineWidth: 0.5 + ) + ) + .shadow(color: .black.opacity(0.3), radius: 12, x: 0, y: 6) .task { await loadThumbnail() } From 1ac0e8d04c5d0bcbbffb6b48427407cdb83daf83 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Tue, 24 Mar 2026 21:14:13 -0400 Subject: [PATCH 06/14] Update all targets and packages to Swift 6.3 Co-Authored-By: Claude Opus 4.6 (1M context) --- Arkavo/Arkavo.xcodeproj/project.pbxproj | 12 ++++++------ .../ArkavoCreator.xcodeproj/project.pbxproj | 16 ++++++++-------- ArkavoKit/Package.swift | 2 +- ArkavoMediaKit/Package.swift | 2 +- HYPERforum/HYPERforum.xcodeproj/project.pbxproj | 12 ++++++------ MuseCore/Package.swift | 2 +- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Arkavo/Arkavo.xcodeproj/project.pbxproj b/Arkavo/Arkavo.xcodeproj/project.pbxproj index 023039aa..4b2f210d 100644 --- a/Arkavo/Arkavo.xcodeproj/project.pbxproj +++ b/Arkavo/Arkavo.xcodeproj/project.pbxproj @@ -1023,7 +1023,7 @@ E5D86638127846B8989BFA94 /* ConnectedAccountsView.swift in Sources */ = {isa = P SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; SUPPORTS_MACCATALYST = NO; SWIFT_EMIT_LOC_STRINGS = YES; - SWIFT_VERSION = 6.2; + SWIFT_VERSION = 6.3; TARGETED_DEVICE_FAMILY = 1; XROS_DEPLOYMENT_TARGET = 2.0; }; @@ -1074,7 +1074,7 @@ E5D86638127846B8989BFA94 /* ConnectedAccountsView.swift in Sources */ = {isa = P SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; SUPPORTS_MACCATALYST = NO; SWIFT_EMIT_LOC_STRINGS = YES; - SWIFT_VERSION = 6.2; + SWIFT_VERSION = 6.3; TARGETED_DEVICE_FAMILY = 1; XROS_DEPLOYMENT_TARGET = 2.0; }; @@ -1096,7 +1096,7 @@ E5D86638127846B8989BFA94 /* ConnectedAccountsView.swift in Sources */ = {isa = P SDKROOT = auto; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 6.2; + SWIFT_VERSION = 6.3; TARGETED_DEVICE_FAMILY = "1,2,7"; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Arkavo.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Arkavo"; XROS_DEPLOYMENT_TARGET = 2.0; @@ -1119,7 +1119,7 @@ E5D86638127846B8989BFA94 /* ConnectedAccountsView.swift in Sources */ = {isa = P SDKROOT = auto; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 6.2; + SWIFT_VERSION = 6.3; TARGETED_DEVICE_FAMILY = "1,2,7"; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Arkavo.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Arkavo"; XROS_DEPLOYMENT_TARGET = 2.0; @@ -1141,7 +1141,7 @@ E5D86638127846B8989BFA94 /* ConnectedAccountsView.swift in Sources */ = {isa = P SDKROOT = auto; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 6.2; + SWIFT_VERSION = 6.3; TARGETED_DEVICE_FAMILY = "1,2,7"; TEST_TARGET_NAME = Arkavo; XROS_DEPLOYMENT_TARGET = 2.0; @@ -1163,7 +1163,7 @@ E5D86638127846B8989BFA94 /* ConnectedAccountsView.swift in Sources */ = {isa = P SDKROOT = auto; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 6.2; + SWIFT_VERSION = 6.3; TARGETED_DEVICE_FAMILY = "1,2,7"; TEST_TARGET_NAME = Arkavo; XROS_DEPLOYMENT_TARGET = 2.0; diff --git a/ArkavoCreator/ArkavoCreator.xcodeproj/project.pbxproj b/ArkavoCreator/ArkavoCreator.xcodeproj/project.pbxproj index 27c992c9..730c2b79 100644 --- a/ArkavoCreator/ArkavoCreator.xcodeproj/project.pbxproj +++ b/ArkavoCreator/ArkavoCreator.xcodeproj/project.pbxproj @@ -362,7 +362,7 @@ SWIFT_APPROACHABLE_CONCURRENCY = NO; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_STRICT_CONCURRENCY = complete; - SWIFT_VERSION = 6.2; + SWIFT_VERSION = 6.3; }; name = Debug; }; @@ -426,7 +426,7 @@ SWIFT_APPROACHABLE_CONCURRENCY = NO; SWIFT_COMPILATION_MODE = wholemodule; SWIFT_STRICT_CONCURRENCY = complete; - SWIFT_VERSION = 6.2; + SWIFT_VERSION = 6.3; }; name = Release; }; @@ -475,7 +475,7 @@ PROVISIONING_PROFILE_SPECIFIER = ""; REGISTER_APP_GROUPS = YES; SWIFT_EMIT_LOC_STRINGS = YES; - SWIFT_VERSION = 6.0; + SWIFT_VERSION = 6.3; }; name = Debug; }; @@ -524,7 +524,7 @@ PROVISIONING_PROFILE_SPECIFIER = ""; REGISTER_APP_GROUPS = YES; SWIFT_EMIT_LOC_STRINGS = YES; - SWIFT_VERSION = 6.0; + SWIFT_VERSION = 6.3; }; name = Release; }; @@ -541,7 +541,7 @@ PRODUCT_BUNDLE_IDENTIFIER = com.arkavo.ArkavoCreatorTests; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 5.0; + SWIFT_VERSION = 6.3; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ArkavoCreator.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/ArkavoCreator"; }; name = Debug; @@ -559,7 +559,7 @@ PRODUCT_BUNDLE_IDENTIFIER = com.arkavo.ArkavoCreatorTests; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 5.0; + SWIFT_VERSION = 6.3; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ArkavoCreator.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/ArkavoCreator"; }; name = Release; @@ -575,7 +575,7 @@ PRODUCT_BUNDLE_IDENTIFIER = com.arkavo.ArkavoCreatorUITests; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 5.0; + SWIFT_VERSION = 6.3; TEST_TARGET_NAME = ArkavoCreator; }; name = Debug; @@ -591,7 +591,7 @@ PRODUCT_BUNDLE_IDENTIFIER = com.arkavo.ArkavoCreatorUITests; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 5.0; + SWIFT_VERSION = 6.3; TEST_TARGET_NAME = ArkavoCreator; }; name = Release; diff --git a/ArkavoKit/Package.swift b/ArkavoKit/Package.swift index 8e5970b7..86a0c1e1 100644 --- a/ArkavoKit/Package.swift +++ b/ArkavoKit/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:6.2 +// swift-tools-version:6.3 import PackageDescription // Shared Swift settings for all targets - enables unused code warnings diff --git a/ArkavoMediaKit/Package.swift b/ArkavoMediaKit/Package.swift index 1946b791..315d7c98 100644 --- a/ArkavoMediaKit/Package.swift +++ b/ArkavoMediaKit/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.2 +// swift-tools-version: 6.3 // The swift-tools-version declares the minimum version of Swift required to build this package. import PackageDescription diff --git a/HYPERforum/HYPERforum.xcodeproj/project.pbxproj b/HYPERforum/HYPERforum.xcodeproj/project.pbxproj index 5b4192d4..4dfbb2df 100644 --- a/HYPERforum/HYPERforum.xcodeproj/project.pbxproj +++ b/HYPERforum/HYPERforum.xcodeproj/project.pbxproj @@ -330,7 +330,7 @@ SUPPORTED_PLATFORMS = macosx; SUPPORTS_MACCATALYST = NO; SWIFT_EMIT_LOC_STRINGS = YES; - SWIFT_VERSION = 6.0; + SWIFT_VERSION = 6.3; }; name = Debug; }; @@ -370,7 +370,7 @@ SUPPORTED_PLATFORMS = macosx; SUPPORTS_MACCATALYST = NO; SWIFT_EMIT_LOC_STRINGS = YES; - SWIFT_VERSION = 6.0; + SWIFT_VERSION = 6.3; }; name = Release; }; @@ -388,7 +388,7 @@ PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 6.0; + SWIFT_VERSION = 6.3; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/HYPERforum.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/HYPERforum"; }; name = Debug; @@ -407,7 +407,7 @@ PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 6.0; + SWIFT_VERSION = 6.3; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/HYPERforum.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/HYPERforum"; }; name = Release; @@ -425,7 +425,7 @@ PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 6.0; + SWIFT_VERSION = 6.3; TEST_TARGET_NAME = HYPERforum; }; name = Debug; @@ -443,7 +443,7 @@ PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; SWIFT_EMIT_LOC_STRINGS = NO; - SWIFT_VERSION = 6.0; + SWIFT_VERSION = 6.3; TEST_TARGET_NAME = HYPERforum; }; name = Release; diff --git a/MuseCore/Package.swift b/MuseCore/Package.swift index 784a9a53..419d1365 100644 --- a/MuseCore/Package.swift +++ b/MuseCore/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.2 +// swift-tools-version: 6.3 import PackageDescription let package = Package( From 0a8991d7c02f746cb8c236f384cb5fb0b2e94643 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Tue, 24 Mar 2026 21:14:25 -0400 Subject: [PATCH 07/14] Add YouTube live streaming with broadcast lifecycle and RTMP keepalive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable YouTube feature flag and implement full broadcast lifecycle: create broadcast, bind stream, transition ready→testing→live, and end on stop. Add network.server entitlement for OAuth callback, silent audio generator for YouTube stream activation, and background RTMP server message handler for ping/pong and window acknowledgements to prevent connection drops. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ArkavoCreator/ArkavoCreator.entitlements | 2 + .../ArkavoCreator/FeatureFlags.swift | 2 +- ArkavoCreator/ArkavoCreator/RecordView.swift | 31 +++ .../ArkavoCreator/StreamViewModel.swift | 35 +++ .../Sources/ArkavoMedia/AudioEncoder.swift | 9 +- .../Sources/ArkavoRecorder/VideoEncoder.swift | 100 ++++++++ .../Sources/ArkavoSocial/YouTubeClient.swift | 225 +++++++++++++++++- .../ArkavoStreaming/RTMP/RTMPPublisher.swift | 77 +++++- 8 files changed, 474 insertions(+), 7 deletions(-) diff --git a/ArkavoCreator/ArkavoCreator/ArkavoCreator.entitlements b/ArkavoCreator/ArkavoCreator/ArkavoCreator.entitlements index 9545ccb1..2f676140 100644 --- a/ArkavoCreator/ArkavoCreator/ArkavoCreator.entitlements +++ b/ArkavoCreator/ArkavoCreator/ArkavoCreator.entitlements @@ -20,6 +20,8 @@ com.apple.security.network.client + com.apple.security.network.server + keychain-access-groups $(AppIdentifierPrefix)com.arkavo.ArkavoCreator diff --git a/ArkavoCreator/ArkavoCreator/FeatureFlags.swift b/ArkavoCreator/ArkavoCreator/FeatureFlags.swift index e844ea69..a45508dc 100644 --- a/ArkavoCreator/ArkavoCreator/FeatureFlags.swift +++ b/ArkavoCreator/ArkavoCreator/FeatureFlags.swift @@ -14,7 +14,7 @@ enum FeatureFlags { /// Arkavo encrypted streaming platform static let arkavoStreaming = false /// YouTube streaming and OAuth integration - static let youtube = false + static let youtube = true /// Patreon patron management static let patreon = false /// Workflow management section diff --git a/ArkavoCreator/ArkavoCreator/RecordView.swift b/ArkavoCreator/ArkavoCreator/RecordView.swift index 10423cac..2de642be 100644 --- a/ArkavoCreator/ArkavoCreator/RecordView.swift +++ b/ArkavoCreator/ArkavoCreator/RecordView.swift @@ -828,6 +828,14 @@ struct RecordView: View { streamKey: "live/creator" ) } else { + // For YouTube, create a broadcast and bind it before starting RTMP + if streamViewModel.selectedPlatform == .youtube { + let broadcastId = try await youtubeClient.createAndBindBroadcast(title: streamViewModel.title) + streamViewModel.youtubeClient = youtubeClient + streamViewModel.youtubeBroadcastId = broadcastId + debugLog("[RecordView] Created YouTube broadcast: \(broadcastId)") + } + try await session.startStreaming(to: destination, streamKey: streamKey) } streamViewModel.isStreaming = true @@ -838,6 +846,29 @@ struct RecordView: View { chatViewModel.connect(twitchClient: twitchClient) withAnimation { showRightPanel = true } } + + // YouTube: transition broadcast to live after RTMP data starts flowing + if streamViewModel.selectedPlatform == .youtube, + let broadcastId = streamViewModel.youtubeBroadcastId { + streamViewModel.youtubeTransitionTask = Task { + // Wait for YouTube to ingest RTMP data and mark stream as active + try? await Task.sleep(for: .seconds(15)) + guard !Task.isCancelled else { return } + for attempt in 1...5 { + guard !Task.isCancelled else { return } + do { + try await youtubeClient.transitionBroadcastToLive(broadcastId: broadcastId) + debugLog("[RecordView] YouTube broadcast transitioned to LIVE") + break + } catch { + debugLog("[RecordView] YouTube transition attempt \(attempt)/5: \(error.localizedDescription)") + if attempt < 5 { + try? await Task.sleep(for: .seconds(10)) + } + } + } + } + } } catch { streamViewModel.error = error.localizedDescription } diff --git a/ArkavoCreator/ArkavoCreator/StreamViewModel.swift b/ArkavoCreator/ArkavoCreator/StreamViewModel.swift index 7d3477c0..c75c73cb 100644 --- a/ArkavoCreator/ArkavoCreator/StreamViewModel.swift +++ b/ArkavoCreator/ArkavoCreator/StreamViewModel.swift @@ -80,6 +80,9 @@ final class StreamViewModel { private var statisticsTimer: Timer? var twitchClient: TwitchAuthClient? + var youtubeClient: YouTubeClient? + var youtubeBroadcastId: String? + var youtubeTransitionTask: Task? private var recordingState = RecordingState.shared // MARK: - Computed Properties @@ -153,6 +156,13 @@ final class StreamViewModel { streamKey: "live/creator" // Default stream key for Arkavo ) } else { + // For YouTube, create a broadcast and bind it before starting RTMP + if selectedPlatform == .youtube, let ytClient = youtubeClient { + let broadcastId = try await ytClient.createAndBindBroadcast(title: title) + youtubeBroadcastId = broadcastId + debugLog("[StreamViewModel] Created YouTube broadcast: \(broadcastId)") + } + // Create RTMP destination for other platforms let destination = RTMPPublisher.Destination( url: effectiveRTMPURL, @@ -171,6 +181,22 @@ final class StreamViewModel { // Start statistics polling startStatisticsTimer() + // For YouTube with enableAutoStart, the broadcast transitions automatically + // when YouTube detects the RTMP stream. If not using autoStart, transition manually: + if selectedPlatform == .youtube, let ytClient = youtubeClient, let broadcastId = youtubeBroadcastId { + Task { + // Wait for YouTube to receive and process the RTMP data + try? await Task.sleep(nanoseconds: 10_000_000_000) // 10 seconds + do { + try await ytClient.transitionBroadcastToLive(broadcastId: broadcastId) + debugLog("[StreamViewModel] YouTube broadcast is now LIVE") + } catch { + // enableAutoStart should handle this, log but don't fail + debugLog("[StreamViewModel] YouTube transition note: \(error.localizedDescription)") + } + } + } + } catch { self.error = error.localizedDescription isConnecting = false @@ -181,6 +207,15 @@ final class StreamViewModel { func stopStreaming() async { guard let session = recordingState.getRecordingSession(), isStreaming else { return } + // Cancel YouTube transition task first, then end broadcast + youtubeTransitionTask?.cancel() + youtubeTransitionTask = nil + if let ytClient = youtubeClient, let broadcastId = youtubeBroadcastId { + try? await ytClient.endBroadcast(broadcastId: broadcastId) + youtubeBroadcastId = nil + debugLog("[StreamViewModel] Ended YouTube broadcast") + } + await session.stopStreaming() isStreaming = false diff --git a/ArkavoKit/Sources/ArkavoMedia/AudioEncoder.swift b/ArkavoKit/Sources/ArkavoMedia/AudioEncoder.swift index 1379c147..ab9e59f3 100644 --- a/ArkavoKit/Sources/ArkavoMedia/AudioEncoder.swift +++ b/ArkavoKit/Sources/ArkavoMedia/AudioEncoder.swift @@ -86,7 +86,12 @@ public final class AudioEncoder: Sendable { /// - Parameters: /// - sampleBuffer: PCM audio sample buffer /// - timestamp: Presentation timestamp + nonisolated(unsafe) private var feedCount = 0 public func feed(_ sampleBuffer: CMSampleBuffer) { + feedCount += 1 + if feedCount == 1 || feedCount % 500 == 0 { + print("🔊 AudioEncoder.feed() called #\(feedCount), accumulated=\(inputBufferFrameCount)/\(targetFrameCount)") + } // Extract PCM data from CMSampleBuffer guard let dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else { print("❌ AudioEncoder: No data buffer") @@ -190,7 +195,9 @@ public final class AudioEncoder: Sendable { onFrame?(frame) - // Removed excessive logging - frame encoding is normal operation + if feedCount <= 3 { + print("🔊 AudioEncoder: Emitted AAC frame \(aacData.count)B at \(timestamp.seconds)s") + } } // Reset buffer for next accumulation diff --git a/ArkavoKit/Sources/ArkavoRecorder/VideoEncoder.swift b/ArkavoKit/Sources/ArkavoRecorder/VideoEncoder.swift index f461979d..ab6e4ca1 100644 --- a/ArkavoKit/Sources/ArkavoRecorder/VideoEncoder.swift +++ b/ArkavoKit/Sources/ArkavoRecorder/VideoEncoder.swift @@ -626,6 +626,7 @@ public actor VideoEncoder { private var audioFrameContinuation: AsyncStream.Continuation? private var videoSendTask: Task? private var audioSendTask: Task? + private var silentAudioTask: Task? /// Start streaming to RTMP destination(s) while recording public func startStreaming(to destination: RTMPPublisher.Destination, streamKey: String) async throws { @@ -750,6 +751,10 @@ public actor VideoEncoder { lastStreamVideoTimestamp = .zero lastStreamAudioTimestamp = .zero + // Start silent audio generator to ensure audio track is always present + // (YouTube requires audio+video to mark a stream as active) + startSilentAudioGenerator(encoder: audioEncoder) + print("✅ RTMP stream started with video and audio encoding") } @@ -759,6 +764,10 @@ public actor VideoEncoder { print("📡 Stopping RTMP stream...") + // Stop silent audio generator + silentAudioTask?.cancel() + silentAudioTask = nil + // Finish the frame queues first videoFrameContinuation?.finish() audioFrameContinuation?.finish() @@ -787,6 +796,97 @@ public actor VideoEncoder { print("✅ RTMP stream stopped") } + /// Generates silent PCM audio and feeds it to the audio encoder. + /// Ensures the RTMP stream always has an audio track (required by YouTube). + /// Real audio from mic/mixer will supplement this; the silent frames + /// act as a fallback when no audio source is active. + private func startSilentAudioGenerator(encoder: ArkavoMedia.AudioEncoder) { + silentAudioTask = Task { [weak self] in + // 48kHz stereo Int16 PCM, 1024 frames per AAC packet + let sampleRate: Double = 48000 + let channels: UInt32 = 2 + let framesPerPacket: Int = 1024 + let bytesPerFrame = Int(channels) * MemoryLayout.size + let bufferSize = framesPerPacket * bytesPerFrame + let silentData = Data(count: bufferSize) // all zeros = silence + let interval = Double(framesPerPacket) / sampleRate // ~21.3ms + + var sampleTime: Double = 0 + + while !Task.isCancelled { + guard let self = self, await self.isStreaming else { break } + + // Only generate silent audio if no real audio is flowing + if await !self.sentAudioSequenceHeader || true { + // Create a CMSampleBuffer with silent PCM data + var formatDesc: CMAudioFormatDescription? + var asbd = AudioStreamBasicDescription( + mSampleRate: sampleRate, + mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, + mBytesPerPacket: UInt32(bytesPerFrame), + mFramesPerPacket: 1, + mBytesPerFrame: UInt32(bytesPerFrame), + mChannelsPerFrame: channels, + mBitsPerChannel: 16, + mReserved: 0 + ) + CMAudioFormatDescriptionCreate( + allocator: kCFAllocatorDefault, + asbd: &asbd, + layoutSize: 0, + layout: nil, + magicCookieSize: 0, + magicCookie: nil, + extensions: nil, + formatDescriptionOut: &formatDesc + ) + + if let formatDesc = formatDesc { + var blockBuffer: CMBlockBuffer? + silentData.withUnsafeBytes { rawPtr in + let ptr = UnsafeMutableRawPointer(mutating: rawPtr.baseAddress!) + CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, + memoryBlock: ptr, + blockLength: bufferSize, + blockAllocator: kCFAllocatorNull, // we manage the memory + customBlockSource: nil, + offsetToData: 0, + dataLength: bufferSize, + flags: 0, + blockBufferOut: &blockBuffer + ) + } + + if let blockBuffer = blockBuffer { + let pts = CMTime(seconds: sampleTime, preferredTimescale: CMTimeScale(sampleRate)) + var sampleBuffer: CMSampleBuffer? + CMAudioSampleBufferCreateReadyWithPacketDescriptions( + allocator: kCFAllocatorDefault, + dataBuffer: blockBuffer, + formatDescription: formatDesc, + sampleCount: framesPerPacket, + presentationTimeStamp: pts, + packetDescriptions: nil, + sampleBufferOut: &sampleBuffer + ) + + if let sampleBuffer = sampleBuffer { + encoder.feed(sampleBuffer) + } + } + } + + sampleTime += interval + } + + try? await Task.sleep(for: .milliseconds(Int(interval * 1000))) + } + } + print("🔇 Silent audio generator started (fallback for YouTube)") + } + // MARK: - NTDF Streaming Methods /// Start NTDF-encrypted streaming to Arkavo diff --git a/ArkavoKit/Sources/ArkavoSocial/YouTubeClient.swift b/ArkavoKit/Sources/ArkavoSocial/YouTubeClient.swift index 026cb9ff..2d0b2198 100644 --- a/ArkavoKit/Sources/ArkavoSocial/YouTubeClient.swift +++ b/ArkavoKit/Sources/ArkavoSocial/YouTubeClient.swift @@ -130,7 +130,7 @@ public actor YouTubeClient: ObservableObject { URLQueryItem(name: "client_id", value: clientId), URLQueryItem(name: "redirect_uri", value: redirectUri), URLQueryItem(name: "response_type", value: "code"), - URLQueryItem(name: "scope", value: "https://www.googleapis.com/auth/youtube.readonly https://www.googleapis.com/auth/youtube.upload https://www.googleapis.com/auth/youtube.force-ssl"), + URLQueryItem(name: "scope", value: "https://www.googleapis.com/auth/youtube https://www.googleapis.com/auth/youtube.force-ssl"), URLQueryItem(name: "access_type", value: "offline"), URLQueryItem(name: "state", value: state), URLQueryItem(name: "code_challenge", value: challenge), @@ -499,10 +499,216 @@ public actor YouTubeClient: ObservableObject { throw YouTubeError.httpError(statusCode: httpResponse.statusCode) } } + + // MARK: - Broadcast Lifecycle + + /// Creates a broadcast, binds it to a stream, and returns the broadcast ID. + /// Call this before starting RTMP streaming to YouTube. + public func createAndBindBroadcast(title: String) async throws -> String { + // 1. Get or create a live stream + let url = URL(string: "https://www.googleapis.com/youtube/v3/liveStreams?part=cdn,snippet&mine=true")! + let listRequest = try await makeAuthorizedRequest(url: url) + let (listData, listResponse) = try await URLSession.shared.data(for: listRequest) + + guard let listHttp = listResponse as? HTTPURLResponse, listHttp.statusCode == 200 else { + throw YouTubeError.googleError("Failed to list live streams") + } + + let streamResponse = try JSONDecoder().decode(YouTubeLiveStreamResponse.self, from: listData) + let streamId: String + if let existing = streamResponse.items.first { + streamId = existing.id + } else { + streamId = try await createLiveStreamAndReturnId() + } + + // 2. Create a broadcast + let broadcastURL = URL(string: "https://www.googleapis.com/youtube/v3/liveBroadcasts?part=snippet,contentDetails,status")! + var broadcastRequest = try await makeAuthorizedRequest(url: broadcastURL) + broadcastRequest.httpMethod = "POST" + + let now = ISO8601DateFormatter().string(from: Date().addingTimeInterval(10)) + let broadcastBody: [String: Any] = [ + "snippet": [ + "title": title.isEmpty ? "Arkavo Creator Live" : title, + "scheduledStartTime": now + ], + "contentDetails": [ + "enableAutoStart": false, + "enableAutoStop": true + ], + "status": [ + "privacyStatus": "public" + ] + ] + broadcastRequest.httpBody = try JSONSerialization.data(withJSONObject: broadcastBody) + + let (broadcastData, broadcastResponse) = try await URLSession.shared.data(for: broadcastRequest) + guard let broadcastHttp = broadcastResponse as? HTTPURLResponse, + (200...201).contains(broadcastHttp.statusCode) else { + if let errorResponse = try? JSONDecoder().decode(GoogleErrorResponse.self, from: broadcastData) { + throw YouTubeError.googleError("Broadcast creation failed: \(errorResponse.error_description ?? errorResponse.error)") + } + let code = (broadcastResponse as? HTTPURLResponse)?.statusCode ?? 0 + throw YouTubeError.googleError("Broadcast creation failed (HTTP \(code))") + } + + let broadcast = try JSONDecoder().decode(YouTubeBroadcastResponse.self, from: broadcastData) + let broadcastId = broadcast.id + + // 3. Bind the stream to the broadcast + let bindURL = URL(string: "https://www.googleapis.com/youtube/v3/liveBroadcasts/bind?id=\(broadcastId)&part=id,contentDetails&streamId=\(streamId)")! + var bindRequest = try await makeAuthorizedRequest(url: bindURL) + bindRequest.httpMethod = "POST" + bindRequest.httpBody = Data() // empty body required + + let (_, bindResponse) = try await URLSession.shared.data(for: bindRequest) + guard let bindHttp = bindResponse as? HTTPURLResponse, bindHttp.statusCode == 200 else { + throw YouTubeError.googleError("Failed to bind stream to broadcast") + } + + return broadcastId + } + + /// Check the current lifecycle status of a broadcast + public func getBroadcastStatus(broadcastId: String) async throws -> String { + let url = URL(string: "https://www.googleapis.com/youtube/v3/liveBroadcasts?id=\(broadcastId)&part=status")! + let request = try await makeAuthorizedRequest(url: url) + let (data, response) = try await URLSession.shared.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 200 else { + throw YouTubeError.googleError("Failed to get broadcast status") + } + + struct BroadcastListResponse: Codable { + let items: [YouTubeBroadcastResponse] + } + let listResponse = try JSONDecoder().decode(BroadcastListResponse.self, from: data) + return listResponse.items.first?.status?.lifeCycleStatus ?? "unknown" + } + + /// Transitions a broadcast to "live" status via testing → live. + /// Waits for the broadcast to reach "ready" state first. + public func transitionBroadcastToLive(broadcastId: String) async throws { + // Wait for broadcast to reach "ready" state (YouTube verifies the stream) + for i in 1...12 { + let status = try await getBroadcastStatus(broadcastId: broadcastId) + print("[YouTubeClient] Broadcast status: \(status) (check \(i)/12)") + if status == "ready" || status == "testing" || status == "live" { + break + } + if i == 12 { + throw YouTubeError.googleError("Broadcast stuck in '\(status)' state. YouTube may not be receiving audio+video.") + } + try await Task.sleep(for: .seconds(5)) + } + + // Transition: ready → testing + let currentStatus = try await getBroadcastStatus(broadcastId: broadcastId) + if currentStatus == "ready" { + try await transitionBroadcast(broadcastId: broadcastId, to: "testing") + // Wait for testing state to be confirmed + try await Task.sleep(for: .seconds(5)) + } + + // Transition: testing → live (skip if already live) + let afterTesting = try await getBroadcastStatus(broadcastId: broadcastId) + if afterTesting == "testing" { + try await transitionBroadcast(broadcastId: broadcastId, to: "live") + } + } + + /// Transition a broadcast to a specific status + private func transitionBroadcast(broadcastId: String, to status: String) async throws { + let url = URL(string: "https://www.googleapis.com/youtube/v3/liveBroadcasts/transition?broadcastStatus=\(status)&id=\(broadcastId)&part=status")! + var request = try await makeAuthorizedRequest(url: url) + request.httpMethod = "POST" + request.httpBody = Data() + + let (data, response) = try await URLSession.shared.data(for: request) + guard let httpResponse = response as? HTTPURLResponse else { + throw YouTubeError.invalidResponse + } + + // Log the full response for debugging + if httpResponse.statusCode != 200 { + let body = String(data: data, encoding: .utf8) ?? "no body" + print("[YouTubeClient] Transition to '\(status)' failed (HTTP \(httpResponse.statusCode)): \(body)") + } + + // 412 means stream isn't active yet — caller should retry + if httpResponse.statusCode == 412 { + throw YouTubeError.googleError("Stream not active yet for '\(status)' transition.") + } + + guard httpResponse.statusCode == 200 else { + // Parse YouTube API v3 error format + if let apiError = try? JSONDecoder().decode(YouTubeAPIError.self, from: data), + let reason = apiError.error.errors.first?.reason { + throw YouTubeError.googleError("Transition to '\(status)': \(reason) - \(apiError.error.message)") + } + if let errorResponse = try? JSONDecoder().decode(GoogleErrorResponse.self, from: data) { + throw YouTubeError.googleError("Transition to '\(status)': \(errorResponse.error_description ?? errorResponse.error)") + } + throw YouTubeError.httpError(statusCode: httpResponse.statusCode) + } + + print("[YouTubeClient] Broadcast transitioned to '\(status)'") + } + + /// Ends a broadcast by transitioning to "complete". + public func endBroadcast(broadcastId: String) async throws { + let url = URL(string: "https://www.googleapis.com/youtube/v3/liveBroadcasts/transition?broadcastStatus=complete&id=\(broadcastId)&part=status")! + var request = try await makeAuthorizedRequest(url: url) + request.httpMethod = "POST" + request.httpBody = Data() + + let (_, response) = try await URLSession.shared.data(for: request) + guard let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 200 else { + // Best-effort — don't throw on end + return + } + } + + /// Creates a live stream and returns its ID (not just the stream key) + private func createLiveStreamAndReturnId() async throws -> String { + let url = URL(string: "https://www.googleapis.com/youtube/v3/liveStreams?part=snippet,cdn,contentDetails")! + var request = try await makeAuthorizedRequest(url: url) + request.httpMethod = "POST" + + let requestBody: [String: Any] = [ + "snippet": ["title": "Arkavo Creator Stream"], + "cdn": [ + "ingestionType": "rtmp", + "frameRate": "30fps", + "resolution": "1080p" + ], + "contentDetails": ["isReusable": true] + ] + request.httpBody = try JSONSerialization.data(withJSONObject: requestBody) + + let (data, response) = try await URLSession.shared.data(for: request) + guard let httpResponse = response as? HTTPURLResponse, + (200...201).contains(httpResponse.statusCode) else { + throw YouTubeError.googleError("Failed to create live stream") + } + + let stream = try JSONDecoder().decode(YouTubeLiveStreamResponse.LiveStream.self, from: data) + return stream.id + } } // MARK: - Supporting Types +struct YouTubeBroadcastResponse: Codable { + let id: String + let status: Status? + + struct Status: Codable { + let lifeCycleStatus: String? + } +} + struct YouTubeLiveStreamResponse: Codable { let items: [LiveStream] @@ -526,6 +732,23 @@ struct YouTubeLiveStreamResponse: Codable { } } +/// YouTube API v3 error response format +struct YouTubeAPIError: Codable { + let error: ErrorBody + + struct ErrorBody: Codable { + let code: Int + let message: String + let errors: [ErrorDetail] + + struct ErrorDetail: Codable { + let message: String + let domain: String + let reason: String + } + } +} + public struct YouTubeChannelInfo { public let id: String public let title: String diff --git a/ArkavoKit/Sources/ArkavoStreaming/RTMP/RTMPPublisher.swift b/ArkavoKit/Sources/ArkavoStreaming/RTMP/RTMPPublisher.swift index 1c07930c..8230fb47 100644 --- a/ArkavoKit/Sources/ArkavoStreaming/RTMP/RTMPPublisher.swift +++ b/ArkavoKit/Sources/ArkavoStreaming/RTMP/RTMPPublisher.swift @@ -210,6 +210,9 @@ public actor RTMPPublisher { state = .publishing startTime = Date() + // Start background handler for server messages (pings, acks, etc.) + startServerMessageHandler() + print("✅ RTMP publishing started") } @@ -1257,14 +1260,80 @@ public actor RTMPPublisher { return (lastReceivedMessageType, payload, totalBytes) } - /// Server messages are handled by the background handler (handleServerMessages). + /// Start background task to read and handle server messages during streaming + /// (ping requests, ack requests, user control messages, etc.) + private func startServerMessageHandler() { + serverMessageTask = Task { [weak self] in + guard let self = self else { return } + print("📡 Server message handler started") + while !Task.isCancelled { + do { + guard await self.state == .publishing else { break } + let (messageType, messageData, messageBytes) = try await self.receiveRTMPMessage() + await self.addBytesReceived(UInt64(messageBytes)) + + switch messageType { + case 1: // Set Chunk Size + if messageData.count >= 4 { + let chunkSize = UInt32(messageData[0]) << 24 | UInt32(messageData[1]) << 16 | + UInt32(messageData[2]) << 8 | UInt32(messageData[3]) + await self.setReceiveChunkSize(Int(chunkSize)) + print("📥 [BG] Server Set Chunk Size: \(chunkSize)") + } + case 3: // Acknowledgement + print("📥 [BG] Server Acknowledgement") + case 4: // User Control (includes ping) + try await self.handleUserControlMessage(messageData) + case 5: // Window Acknowledgement Size + if messageData.count >= 4 { + let size = UInt32(messageData[0]) << 24 | UInt32(messageData[1]) << 16 | + UInt32(messageData[2]) << 8 | UInt32(messageData[3]) + await self.setServerWindowAckSize(size) + print("📥 [BG] Server Window Ack Size: \(size)") + } + case 6: // Set Peer Bandwidth + print("📥 [BG] Server Set Peer Bandwidth") + case 20: // AMF0 Command + print("📥 [BG] Server AMF0 command (ignored during streaming)") + default: + print("📥 [BG] Server message type \(messageType) (\(messageData.count) bytes)") + } + + // Send acknowledgement if we've received enough bytes + let received = await self.getBytesReceived() + let ackSize = await self.getServerWindowAckSize() + let lastAck = await self.getLastAckSent() + if received - lastAck >= UInt64(ackSize) { + try await self.sendWindowAcknowledgement(bytesReceived: UInt32(received & 0xFFFFFFFF)) + await self.setLastAckSent(received) + } + } catch is CancellationError { + break + } catch { + if !Task.isCancelled { + print("⚠️ [BG] Server message handler error: \(error.localizedDescription)") + } + break + } + } + print("📡 Server message handler stopped") + } + } + + // Actor-isolated accessors for server message handler + private func addBytesReceived(_ bytes: UInt64) { bytesReceived += bytes } + private func getBytesReceived() -> UInt64 { bytesReceived } + private func getServerWindowAckSize() -> UInt32 { serverWindowAckSize } + private func setServerWindowAckSize(_ size: UInt32) { serverWindowAckSize = size } + private func getLastAckSent() -> UInt64 { lastAckSent } + private func setLastAckSent(_ value: UInt64) { lastAckSent = value } + private func setReceiveChunkSize(_ size: Int) { receiveChunkSize = size } + + /// Server messages are handled by the background handler (startServerMessageHandler). /// This method is kept for the CMSampleBuffer API (publishVideo/publishAudio) but is /// now a no-op since the background handler processes all server messages. private func processAllPendingServerMessages() async throws { // No-op: background handler processes server messages via blocking receive. - // Using minimumIncompleteLength: 0 for non-blocking reads poisons NWConnection's - // read queue ("already delivered final read"), so all reads go through the - // background handler's blocking receiveRTMPChunk() instead. } /// Receive and parse an RTMP chunk from the server From f75d9641308f5fa058cc37af73008c129de9a2bd Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Tue, 24 Mar 2026 22:22:14 -0400 Subject: [PATCH 08/14] Add simulcast support: multi-destination RTMP fan-out with unified chat Encode video/audio once, fan out to N RTMPPublisher instances in parallel. Each publisher has independent send tasks, sequence headers, AsyncStream continuations with .bufferingNewest(30) backpressure, and per-destination frame rate limiting to prevent YouTube "faster than realtime" errors. - VideoEncoder: startStreaming(to:) accepts array of destinations - RecordingSession: multi-destination pass-through, per-platform stop - StreamViewModel: selectedPlatforms set, per-platform PlatformConfig - StreamDestinationPicker: multi-select toggle cards, bandwidth estimate - StreamInfoFormView: universal form with platform-specific sections - ChatPanelViewModel: concurrent Twitch IRC + YouTube polling into unified message feed with per-platform connect/disconnect - TwitchAuthClient: token refresh, EventSub scopes, keychain storage - TwitchEventSubClient: WebSocket client for follows, subs, cheers, raids - YouTubeClient: live chat polling via OAuth, Sendable response types Co-Authored-By: Claude Opus 4.6 (1M context) --- ArkavoCreator/ArkavoCreator/RecordView.swift | 61 ++- .../StreamDestinationPicker.swift | 151 ++++-- .../ArkavoCreator/StreamInfoFormView.swift | 324 ++++++++----- .../ArkavoCreator/StreamViewModel.swift | 363 +++++++-------- .../Streaming/ChatPanelViewModel.swift | 180 ++++++-- .../Streaming/TwitchEventSubClient.swift | 424 +++++++++++++++++ .../ArkavoCreator/TwitchAuthClient.swift | 127 ++++- .../ArkavoRecorder/RecordingSession.swift | 23 +- .../Sources/ArkavoRecorder/VideoEncoder.swift | 436 ++++++++++-------- .../Sources/ArkavoSocial/YouTubeClient.swift | 89 ++++ 10 files changed, 1537 insertions(+), 641 deletions(-) create mode 100644 ArkavoCreator/ArkavoCreator/Streaming/TwitchEventSubClient.swift diff --git a/ArkavoCreator/ArkavoCreator/RecordView.swift b/ArkavoCreator/ArkavoCreator/RecordView.swift index 2de642be..4c27f063 100644 --- a/ArkavoCreator/ArkavoCreator/RecordView.swift +++ b/ArkavoCreator/ArkavoCreator/RecordView.swift @@ -804,9 +804,8 @@ struct RecordView: View { // MARK: - Streaming private func startStreaming(destination: RTMPPublisher.Destination, streamKey: String) async { - // Ensure we have an active session (either from recording or create one for streaming) + // Ensure we have an active session if RecordingState.shared.recordingSession == nil { - // Start a preview-mode session for streaming without recording await viewModel.startPreviewSession() } @@ -816,42 +815,66 @@ struct RecordView: View { return } - // Check if this is Arkavo (NTDF-encrypted streaming) - if streamViewModel.selectedPlatform == .arkavo { + let selectedPlatforms = streamViewModel.selectedPlatforms + + // Handle Arkavo NTDF separately + if selectedPlatforms.contains(.arkavo) { guard let kasURL = URL(string: "https://100.arkavo.net") else { streamViewModel.error = "Invalid KAS URL" return } try await session.startNTDFStreaming( kasURL: kasURL, - rtmpURL: destination.url, + rtmpURL: StreamViewModel.StreamPlatform.arkavo.rtmpURL, streamKey: "live/creator" ) - } else { - // For YouTube, create a broadcast and bind it before starting RTMP - if streamViewModel.selectedPlatform == .youtube { + } + + // Build RTMP destinations for all non-Arkavo platforms + let rtmpPlatforms = selectedPlatforms.filter { !$0.isEncrypted } + if !rtmpPlatforms.isEmpty { + // YouTube: create broadcast before RTMP + if rtmpPlatforms.contains(.youtube) { let broadcastId = try await youtubeClient.createAndBindBroadcast(title: streamViewModel.title) - streamViewModel.youtubeClient = youtubeClient - streamViewModel.youtubeBroadcastId = broadcastId + streamViewModel.platformConfigs[.youtube, default: StreamViewModel.PlatformConfig()].broadcastId = broadcastId debugLog("[RecordView] Created YouTube broadcast: \(broadcastId)") } - try await session.startStreaming(to: destination, streamKey: streamKey) + // Build destinations array + var destinations: [(id: String, destination: RTMPPublisher.Destination, streamKey: String)] = [] + for platform in rtmpPlatforms { + let config = streamViewModel.platformConfigs[platform] ?? StreamViewModel.PlatformConfig() + let url = platform == .custom ? streamViewModel.customRTMPURL : platform.rtmpURL + let dest = RTMPPublisher.Destination(url: url, platform: platform.rawValue.lowercased()) + var key = config.streamKey + if platform == .twitch && streamViewModel.isBandwidthTest { + key += "?bandwidthtest=true" + } + destinations.append((id: platform.rawValue.lowercased(), destination: dest, streamKey: key)) + } + + try await session.startStreaming(destinations: destinations) } + streamViewModel.isStreaming = true streamViewModel.startStatisticsPolling() - // Auto-connect Twitch chat - if streamViewModel.selectedPlatform == .twitch { - chatViewModel.connect(twitchClient: twitchClient) + // Auto-connect chat for all selected platforms (unified feed) + if selectedPlatforms.contains(.twitch) && twitchClient.isAuthenticated { + chatViewModel.connectTwitch(twitchClient: twitchClient) + } + if selectedPlatforms.contains(.youtube), + let broadcastId = streamViewModel.platformConfigs[.youtube]?.broadcastId { + chatViewModel.connectYouTube(youtubeClient: youtubeClient, broadcastId: broadcastId) + } + if selectedPlatforms.contains(.twitch) || selectedPlatforms.contains(.youtube) { withAnimation { showRightPanel = true } } - // YouTube: transition broadcast to live after RTMP data starts flowing - if streamViewModel.selectedPlatform == .youtube, - let broadcastId = streamViewModel.youtubeBroadcastId { - streamViewModel.youtubeTransitionTask = Task { - // Wait for YouTube to ingest RTMP data and mark stream as active + // YouTube: transition broadcast to live + if selectedPlatforms.contains(.youtube), + let broadcastId = streamViewModel.platformConfigs[.youtube]?.broadcastId { + streamViewModel.platformConfigs[.youtube]?.transitionTask = Task { try? await Task.sleep(for: .seconds(15)) guard !Task.isCancelled else { return } for attempt in 1...5 { diff --git a/ArkavoCreator/ArkavoCreator/StreamDestinationPicker.swift b/ArkavoCreator/ArkavoCreator/StreamDestinationPicker.swift index 443a544c..87d21942 100644 --- a/ArkavoCreator/ArkavoCreator/StreamDestinationPicker.swift +++ b/ArkavoCreator/ArkavoCreator/StreamDestinationPicker.swift @@ -16,20 +16,22 @@ struct StreamDestinationPicker: View { private var arkavoAuthState: ArkavoAuthState { ArkavoAuthState.shared } - /// Whether the stream info step is available (Twitch + authenticated) + /// Whether the stream info step is available private var hasStreamInfoStep: Bool { - streamViewModel.selectedPlatform == .twitch && twitchClient.isAuthenticated + (streamViewModel.selectedPlatform == .twitch && twitchClient.isAuthenticated) || + (streamViewModel.selectedPlatform == .youtube && youtubeClient.isAuthenticated) } var body: some View { Group { if showStreamInfo { StreamInfoFormView( + platform: streamViewModel.selectedPlatform, twitchClient: twitchClient, + youtubeClient: youtubeClient, onBack: { showStreamInfo = false }, onStartStream: { await startStream() - // Dismiss handled inside startStream } ) .padding(24) @@ -73,21 +75,39 @@ struct StreamDestinationPicker: View { }) { platform in PlatformCard( platform: platform, - isSelected: streamViewModel.selectedPlatform == platform, + isSelected: streamViewModel.selectedPlatforms.contains(platform), action: { - streamViewModel.selectedPlatform = platform + // Toggle multi-select + if streamViewModel.selectedPlatforms.contains(platform) { + // Don't allow deselecting the last platform + if streamViewModel.selectedPlatforms.count > 1 { + streamViewModel.selectedPlatforms.remove(platform) + } + } else { + streamViewModel.selectedPlatforms.insert(platform) + } streamViewModel.loadStreamKey() } ) } } + + if streamViewModel.selectedPlatforms.count > 1 { + Text("Simulcast: \(streamViewModel.estimatedTotalBitrate) estimated upload") + .font(.caption) + .foregroundStyle(.secondary) + } } - // Stream Key / Auth Section - if streamViewModel.selectedPlatform == .twitch && !twitchClient.isAuthenticated { - twitchConnectSection - } else { - streamKeySection + // Stream Key / Auth Section — per selected platform + ForEach(Array(streamViewModel.selectedPlatforms).sorted(by: { $0.rawValue < $1.rawValue }), id: \.self) { platform in + if platform == .twitch && !twitchClient.isAuthenticated { + twitchConnectSection + } else if platform == .youtube && !youtubeClient.isAuthenticated { + youtubeConnectSection + } else if platform.requiresStreamKey { + streamKeySection(for: platform) + } } // Custom RTMP URL (if custom platform) @@ -217,16 +237,59 @@ struct StreamDestinationPicker: View { .padding(.vertical, 8) } - // MARK: - Stream Key Input (authenticated) + // MARK: - YouTube Connect (unauthenticated) + + private var youtubeConnectSection: some View { + VStack(spacing: 12) { + Image(systemName: "play.rectangle.fill") + .font(.system(size: 36)) + .foregroundStyle(.red) + + Text("Connect your YouTube account to go live") + .font(.subheadline) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + + Button { + Task { + do { + try await youtubeClient.authenticateWithLocalServer() + } catch { + debugLog("YouTube OAuth error: \(error)") + } + } + } label: { + HStack { + Image(systemName: "person.crop.circle.badge.plus") + Text("Connect to YouTube") + } + .font(.headline) + .frame(maxWidth: .infinity) + .padding(.vertical, 12) + .background(Color.red) + .foregroundColor(.white) + .cornerRadius(10) + } + .buttonStyle(.plain) + } + .padding(.vertical, 8) + } + + // MARK: - Stream Key Input (per platform) - private var streamKeySection: some View { - VStack(alignment: .leading, spacing: 8) { - Text("Stream Key") + private func streamKeySection(for platform: StreamViewModel.StreamPlatform) -> some View { + let keyBinding = Binding( + get: { streamViewModel.platformConfigs[platform]?.streamKey ?? "" }, + set: { streamViewModel.platformConfigs[platform, default: StreamViewModel.PlatformConfig()].streamKey = $0 } + ) + + return VStack(alignment: .leading, spacing: 8) { + Text("\(platform.rawValue) Stream Key") .font(.headline) .foregroundStyle(.secondary) HStack { - SecureField("Enter your stream key", text: $streamViewModel.streamKey) + SecureField("Enter your stream key", text: keyBinding) .textFieldStyle(.plain) .padding(12) .background(.background.opacity(0.5)) @@ -236,7 +299,7 @@ struct StreamDestinationPicker: View { .stroke(.white.opacity(0.2), lineWidth: 1) ) - if streamViewModel.selectedPlatform == .twitch && twitchClient.isAuthenticated { + if platform == .twitch && twitchClient.isAuthenticated { Button { Task { await fetchTwitchStreamKey() } } label: { @@ -249,35 +312,21 @@ struct StreamDestinationPicker: View { .help("Fetch stream key from Twitch") } - if streamViewModel.selectedPlatform == .youtube { + if platform == .youtube && youtubeClient.isAuthenticated { Button { - Task { - if youtubeClient.isAuthenticated { - await fetchYouTubeStreamKey() - } else { - debugLog("[StreamDestinationPicker] YouTube not authenticated, starting auth flow...") - do { - try await youtubeClient.authenticateWithLocalServer() - await fetchYouTubeStreamKey() - } catch { - await MainActor.run { - streamViewModel.error = "YouTube login failed: \(error.localizedDescription)" - } - } - } - } + Task { await fetchYouTubeStreamKey() } } label: { - Image(systemName: youtubeClient.isAuthenticated ? "arrow.clockwise" : "person.crop.circle.badge.plus") + Image(systemName: "arrow.clockwise") .padding(10) .background(.ultraThinMaterial) .cornerRadius(8) } .buttonStyle(.plain) - .help(youtubeClient.isAuthenticated ? "Fetch stream key from YouTube" : "Login to YouTube to fetch stream key") + .help("Fetch stream key from YouTube") } } - if streamViewModel.selectedPlatform == .twitch && streamViewModel.streamKey.isEmpty { + if platform == .twitch && keyBinding.wrappedValue.isEmpty { if let username = twitchClient.username { Link(destination: URL(string: "https://dashboard.twitch.tv/u/\(username.lowercased())/settings/stream") ?? URL(string: "https://dashboard.twitch.tv")!) { Label("Copy stream key from Twitch Dashboard", systemImage: "arrow.up.right.square") @@ -289,32 +338,34 @@ struct StreamDestinationPicker: View { } private var canStartStream: Bool { - // Block streaming when Twitch is selected but not authenticated - if streamViewModel.selectedPlatform == .twitch && !twitchClient.isAuthenticated { - return false + // Check all selected platforms have what they need + for platform in streamViewModel.selectedPlatforms { + if platform == .twitch && !twitchClient.isAuthenticated { return false } + if platform.requiresStreamKey { + let key = streamViewModel.platformConfigs[platform]?.streamKey ?? "" + if key.isEmpty { return false } + } + if platform == .custom && streamViewModel.customRTMPURL.isEmpty { return false } } - let hasValidKey = !streamViewModel.streamKey.isEmpty - return hasValidKey && - (streamViewModel.selectedPlatform != .custom || !streamViewModel.customRTMPURL.isEmpty) + return !streamViewModel.selectedPlatforms.isEmpty } private func startStream() async { isLoading = true defer { isLoading = false } - // Save the stream key streamViewModel.saveStreamKey() - // Create destination + // Build destination for primary platform (RecordView handles multi-destination) + let primary = streamViewModel.selectedPlatform let destination = RTMPPublisher.Destination( - url: streamViewModel.effectiveRTMPURL, - platform: streamViewModel.selectedPlatform.rawValue.lowercased() + url: primary == .custom ? streamViewModel.customRTMPURL : primary.rtmpURL, + platform: primary.rawValue.lowercased() ) + let key = streamViewModel.platformConfigs[primary]?.streamKey ?? "" - // Start streaming - await onStartStream(destination, streamViewModel.streamKey) + await onStartStream(destination, key) - // Dismiss if successful if streamViewModel.error == nil { dismiss() } @@ -324,7 +375,7 @@ struct StreamDestinationPicker: View { do { if let key = try await twitchClient.fetchStreamKey() { debugLog("[StreamDestinationPicker] Fetched Twitch stream key") - streamViewModel.streamKey = key + streamViewModel.platformConfigs[.twitch, default: StreamViewModel.PlatformConfig()].streamKey = key streamViewModel.saveStreamKey() } else { streamViewModel.error = "Could not fetch stream key — copy it from the Twitch Dashboard" @@ -339,7 +390,7 @@ struct StreamDestinationPicker: View { if let key = try await youtubeClient.fetchStreamKey() { debugLog("[StreamDestinationPicker] Fetched YouTube stream key") await MainActor.run { - streamViewModel.streamKey = key + streamViewModel.platformConfigs[.youtube, default: StreamViewModel.PlatformConfig()].streamKey = key streamViewModel.saveStreamKey() } } diff --git a/ArkavoCreator/ArkavoCreator/StreamInfoFormView.swift b/ArkavoCreator/ArkavoCreator/StreamInfoFormView.swift index c9e030dc..05e6debf 100644 --- a/ArkavoCreator/ArkavoCreator/StreamInfoFormView.swift +++ b/ArkavoCreator/ArkavoCreator/StreamInfoFormView.swift @@ -1,20 +1,31 @@ import SwiftUI +import ArkavoKit -/// Stream info editing form for Twitch (title, category, tags, language, content labels) +/// Universal stream info editing form for Twitch & YouTube. /// Embedded in the StreamDestinationPicker as step 2 of the go-live flow. struct StreamInfoFormView: View { - @ObservedObject var twitchClient: TwitchAuthClient + let platform: StreamViewModel.StreamPlatform - // Stream info fields + // Platform clients (provide the one that matches `platform`) + var twitchClient: TwitchAuthClient? + @ObservedObject var youtubeClient: YouTubeClient + + // Stream info fields (shared) @State var streamTitle: String = "" + @State var tags: [String] = [] + @State var language: String = "en" + + // Twitch-specific @State var goLiveNotification: String = "" @State var categoryName: String = "" @State var categoryId: String = "" - @State var tags: [String] = [] - @State var language: String = "en" @State var isRerun: Bool = false @State var isBrandedContent: Bool = false + // YouTube-specific + @State var privacyStatus: String = "public" + @State var youtubeDescription: String = "" + // UI state @State private var newTag: String = "" @State private var categorySearchResults: [TwitchCategory] = [] @@ -31,6 +42,7 @@ struct StreamInfoFormView: View { private static let titleLimit = 140 private static let tagCharLimit = 25 private static let maxTags = 10 + private static let descriptionLimit = 5000 private static let languages: [(code: String, name: String)] = [ ("en", "English"), ("es", "Spanish"), ("fr", "French"), ("de", "German"), @@ -62,25 +74,23 @@ struct StreamInfoFormView: View { Spacer() - // Invisible spacer to balance the back button - Color.clear.frame(width: 50, height: 1) + // Platform badge + Text(platform.rawValue) + .font(.caption.weight(.medium)) + .padding(.horizontal, 8) + .padding(.vertical, 3) + .background(platformColor.opacity(0.2)) + .foregroundStyle(platformColor) + .cornerRadius(6) } .padding(.bottom, 16) // Scrollable form ScrollView { VStack(alignment: .leading, spacing: 20) { - // Title + // Title (universal) fieldSection(label: "Title", counter: "\(streamTitle.count)/\(Self.titleLimit)") { - TextField("Stream title", text: $streamTitle) - .textFieldStyle(.plain) - .padding(10) - .background(.background.opacity(0.5)) - .cornerRadius(8) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(.white.opacity(0.2), lineWidth: 1) - ) + styledTextField("Stream title", text: $streamTitle) .onChange(of: streamTitle) { _, newValue in if newValue.count > Self.titleLimit { streamTitle = String(newValue.prefix(Self.titleLimit)) @@ -88,92 +98,105 @@ struct StreamInfoFormView: View { } } - // Go Live Notification - fieldSection(label: "Go Live Notification", counter: "\(goLiveNotification.count)/\(Self.titleLimit)") { - TextField("Notification text for followers", text: $goLiveNotification) - .textFieldStyle(.plain) - .padding(10) - .background(.background.opacity(0.5)) - .cornerRadius(8) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(.white.opacity(0.2), lineWidth: 1) - ) - .onChange(of: goLiveNotification) { _, newValue in - if newValue.count > Self.titleLimit { - goLiveNotification = String(newValue.prefix(Self.titleLimit)) + // Twitch: Go Live Notification + if platform == .twitch { + fieldSection(label: "Go Live Notification", counter: "\(goLiveNotification.count)/\(Self.titleLimit)") { + styledTextField("Notification text for followers", text: $goLiveNotification) + .onChange(of: goLiveNotification) { _, newValue in + if newValue.count > Self.titleLimit { + goLiveNotification = String(newValue.prefix(Self.titleLimit)) + } } - } + } } - // Category - fieldSection(label: "Category") { - VStack(alignment: .leading, spacing: 4) { - HStack { - TextField("Search categories", text: $categoryName) - .textFieldStyle(.plain) - .padding(10) - .background(.background.opacity(0.5)) - .cornerRadius(8) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(.white.opacity(0.2), lineWidth: 1) - ) - .onChange(of: categoryName) { _, newValue in - debouncedCategorySearch(query: newValue) + // YouTube: Description + if platform == .youtube { + fieldSection(label: "Description", counter: "\(youtubeDescription.count)/\(Self.descriptionLimit)") { + TextEditor(text: $youtubeDescription) + .font(.body) + .frame(minHeight: 60, maxHeight: 100) + .padding(6) + .background(.background.opacity(0.5)) + .cornerRadius(8) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(.white.opacity(0.2), lineWidth: 1) + ) + .onChange(of: youtubeDescription) { _, newValue in + if newValue.count > Self.descriptionLimit { + youtubeDescription = String(newValue.prefix(Self.descriptionLimit)) } - - if isSearchingCategories { - ProgressView() - .scaleEffect(0.7) } - } + } + } - if showCategoryResults && !categorySearchResults.isEmpty { - VStack(spacing: 0) { - ForEach(categorySearchResults) { category in - Button { - categoryName = category.name - categoryId = category.id - showCategoryResults = false - categorySearchResults = [] - } label: { - Text(category.name) - .frame(maxWidth: .infinity, alignment: .leading) - .padding(.horizontal, 10) - .padding(.vertical, 6) - .contentShape(Rectangle()) + // Twitch: Category search + if platform == .twitch { + fieldSection(label: "Category") { + VStack(alignment: .leading, spacing: 4) { + HStack { + styledTextField("Search categories", text: $categoryName) + .onChange(of: categoryName) { _, newValue in + debouncedCategorySearch(query: newValue) } - .buttonStyle(.plain) - if category.id != categorySearchResults.last?.id { - Divider().opacity(0.3) + if isSearchingCategories { + ProgressView() + .scaleEffect(0.7) + } + } + + if showCategoryResults && !categorySearchResults.isEmpty { + VStack(spacing: 0) { + ForEach(categorySearchResults) { category in + Button { + categoryName = category.name + categoryId = category.id + showCategoryResults = false + categorySearchResults = [] + } label: { + Text(category.name) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 10) + .padding(.vertical, 6) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + + if category.id != categorySearchResults.last?.id { + Divider().opacity(0.3) + } } } + .background(.background.opacity(0.8)) + .cornerRadius(8) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(.white.opacity(0.15), lineWidth: 1) + ) } - .background(.background.opacity(0.8)) - .cornerRadius(8) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(.white.opacity(0.15), lineWidth: 1) - ) } } } - // Tags + // YouTube: Privacy + if platform == .youtube { + fieldSection(label: "Privacy") { + Picker("", selection: $privacyStatus) { + Text("Public").tag("public") + Text("Unlisted").tag("unlisted") + Text("Private").tag("private") + } + .pickerStyle(.segmented) + } + } + + // Tags (universal) fieldSection(label: "Tags", counter: "\(tags.count)/\(Self.maxTags)") { VStack(alignment: .leading, spacing: 8) { HStack { - TextField("Add a tag", text: $newTag) - .textFieldStyle(.plain) - .padding(10) - .background(.background.opacity(0.5)) - .cornerRadius(8) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(.white.opacity(0.2), lineWidth: 1) - ) + styledTextField("Add a tag", text: $newTag) .onChange(of: newTag) { _, newValue in if newValue.count > Self.tagCharLimit { newTag = String(newValue.prefix(Self.tagCharLimit)) @@ -191,7 +214,7 @@ struct StreamInfoFormView: View { .disabled(newTag.isEmpty || tags.count >= Self.maxTags) } - Text("Up to \(Self.maxTags) tags. Each tag max \(Self.tagCharLimit) characters, no spaces or special characters.") + Text("Up to \(Self.maxTags) tags. Each tag max \(Self.tagCharLimit) characters.") .font(.caption2) .foregroundStyle(.secondary) @@ -219,7 +242,7 @@ struct StreamInfoFormView: View { } } - // Language + // Language (universal) fieldSection(label: "Stream Language") { Picker("", selection: $language) { ForEach(Self.languages, id: \.code) { lang in @@ -229,20 +252,22 @@ struct StreamInfoFormView: View { .labelsHidden() } - // Content Classification - fieldSection(label: "Content Classification") { - VStack(alignment: .leading, spacing: 8) { - Toggle("Rerun", isOn: $isRerun) - .font(.subheadline) - Text("Let viewers know your stream was previously recorded.") - .font(.caption2) - .foregroundStyle(.secondary) - - Toggle("Branded Content", isOn: $isBrandedContent) - .font(.subheadline) - Text("Let viewers know if your stream features branded content.") - .font(.caption2) - .foregroundStyle(.secondary) + // Twitch: Content Classification + if platform == .twitch { + fieldSection(label: "Content Classification") { + VStack(alignment: .leading, spacing: 8) { + Toggle("Rerun", isOn: $isRerun) + .font(.subheadline) + Text("Let viewers know your stream was previously recorded.") + .font(.caption2) + .foregroundStyle(.secondary) + + Toggle("Branded Content", isOn: $isBrandedContent) + .font(.subheadline) + Text("Let viewers know if your stream features branded content.") + .font(.caption2) + .foregroundStyle(.secondary) + } } } } @@ -252,7 +277,7 @@ struct StreamInfoFormView: View { if needsReauth { HStack { Image(systemName: "exclamationmark.triangle") - Text("Please reconnect Twitch to update stream info.") + Text("Please reconnect \(platform.rawValue) to update stream info.") .font(.caption) } .foregroundStyle(.orange) @@ -301,11 +326,33 @@ struct StreamInfoFormView: View { .disabled(isSaving) .padding(.top, 12) } - .onAppear { loadFromTwitch() } + .onAppear { loadFromPlatform() } + } + + // MARK: - Platform color + + private var platformColor: Color { + switch platform { + case .twitch: .purple + case .youtube: .red + default: .blue + } } // MARK: - Helpers + private func styledTextField(_ placeholder: String, text: Binding) -> some View { + TextField(placeholder, text: text) + .textFieldStyle(.plain) + .padding(10) + .background(.background.opacity(0.5)) + .cornerRadius(8) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(.white.opacity(0.2), lineWidth: 1) + ) + } + private func fieldSection(label: String, counter: String? = nil, @ViewBuilder content: () -> Content) -> some View { VStack(alignment: .leading, spacing: 6) { HStack { @@ -340,6 +387,8 @@ struct StreamInfoFormView: View { newTag = "" } + // MARK: - Twitch category search + private func debouncedCategorySearch(query: String) { categorySearchTask?.cancel() guard !query.isEmpty else { @@ -350,11 +399,11 @@ struct StreamInfoFormView: View { categorySearchTask = Task { try? await Task.sleep(for: .milliseconds(300)) - guard !Task.isCancelled else { return } + guard !Task.isCancelled, let twitch = twitchClient else { return } isSearchingCategories = true do { - let results = try await twitchClient.searchCategories(query: query) + let results = try await twitch.searchCategories(query: query) if !Task.isCancelled { categorySearchResults = results showCategoryResults = true @@ -368,14 +417,29 @@ struct StreamInfoFormView: View { } } - private func loadFromTwitch() { - streamTitle = twitchClient.channelTitle ?? twitchClient.streamTitle ?? "" - goLiveNotification = "" - categoryName = twitchClient.gameName ?? "" - categoryId = twitchClient.gameId ?? "" - tags = twitchClient.channelTags - language = twitchClient.broadcasterLanguage ?? "en" - isBrandedContent = twitchClient.isBrandedContent + // MARK: - Load / Save + + private func loadFromPlatform() { + switch platform { + case .twitch: + guard let twitch = twitchClient else { return } + streamTitle = twitch.channelTitle ?? twitch.streamTitle ?? "" + goLiveNotification = "" + categoryName = twitch.gameName ?? "" + categoryId = twitch.gameId ?? "" + tags = twitch.channelTags + language = twitch.broadcasterLanguage ?? "en" + isBrandedContent = twitch.isBrandedContent + + case .youtube: + streamTitle = "" + youtubeDescription = "" + privacyStatus = "public" + language = "en" + + default: + break + } } private func saveAndStartStream() async { @@ -383,13 +447,29 @@ struct StreamInfoFormView: View { saveError = nil needsReauth = false - do { - // Build content classification labels - var ccls: [TwitchContentLabel] = [] - // Rerun is not a standard CCL — it's handled differently on Twitch. - // We include branded content via the is_branded_content param. + switch platform { + case .twitch: + await saveTwitchAndStart() + case .youtube: + // YouTube broadcast info is set at creation time; + // title is passed through StreamViewModel.title + await onStartStream() + default: + await onStartStream() + } + + isSaving = false + } + + private func saveTwitchAndStart() async { + guard let twitch = twitchClient else { + await onStartStream() + return + } - try await twitchClient.updateChannelInfo( + do { + let ccls: [TwitchContentLabel] = [] + try await twitch.updateChannelInfo( title: streamTitle.isEmpty ? nil : streamTitle, gameId: categoryId.isEmpty ? nil : categoryId, language: language, @@ -397,19 +477,13 @@ struct StreamInfoFormView: View { contentClassificationLabels: ccls.isEmpty ? nil : ccls, isBrandedContent: isBrandedContent ) - - // Success — now start the stream await onStartStream() } catch TwitchError.scopeRequired { needsReauth = true - // Still allow streaming even if update fails await onStartStream() } catch { saveError = "Failed to update stream info: \(error.localizedDescription)" - // Still allow streaming even if update fails await onStartStream() } - - isSaving = false } } diff --git a/ArkavoCreator/ArkavoCreator/StreamViewModel.swift b/ArkavoCreator/ArkavoCreator/StreamViewModel.swift index c75c73cb..e0e899d1 100644 --- a/ArkavoCreator/ArkavoCreator/StreamViewModel.swift +++ b/ArkavoCreator/ArkavoCreator/StreamViewModel.swift @@ -1,7 +1,6 @@ import SwiftUI import ArkavoKit import ArkavoStreaming -import ArkavoKit @Observable @MainActor @@ -9,7 +8,7 @@ final class StreamViewModel { // MARK: - Stream Configuration - enum StreamPlatform: String, CaseIterable, Identifiable { + enum StreamPlatform: String, CaseIterable, Identifiable, Hashable { case arkavo = "Arkavo" case twitch = "Twitch" case youtube = "YouTube" @@ -19,51 +18,46 @@ final class StreamViewModel { var rtmpURL: String { switch self { - case .arkavo: - return "rtmp://100.arkavo.net:1935" - case .twitch: - return "rtmp://live.twitch.tv/app" - case .youtube: - return "rtmp://a.rtmp.youtube.com/live2" - case .custom: - return "" + case .arkavo: "rtmp://100.arkavo.net:1935" + case .twitch: "rtmp://live.twitch.tv/app" + case .youtube: "rtmp://a.rtmp.youtube.com/live2" + case .custom: "" } } var requiresStreamKey: Bool { - switch self { - case .arkavo: - return false // Arkavo uses authenticated session, not stream key - default: - return true - } + self != .arkavo } var icon: String { switch self { - case .arkavo: - return "lock.shield" - case .twitch: - return "tv" - case .youtube: - return "play.rectangle" - case .custom: - return "server.rack" + case .arkavo: "lock.shield" + case .twitch: "tv" + case .youtube: "play.rectangle" + case .custom: "server.rack" } } - var isEncrypted: Bool { - self == .arkavo - } + var isEncrypted: Bool { self == .arkavo } + } + + // MARK: - Per-Platform Config + + struct PlatformConfig { + var streamKey: String = "" + var broadcastId: String? + var transitionTask: Task? + var error: String? + var isLive: Bool = false } // MARK: - State - var selectedPlatform: StreamPlatform = .twitch + var selectedPlatforms: Set = [.twitch] + var platformConfigs: [StreamPlatform: PlatformConfig] = [:] var customRTMPURL: String = "" - var streamKey: String = "" var title: String = "" - var isBandwidthTest: Bool = false // Twitch bandwidth test mode + var isBandwidthTest: Bool = false var isStreaming: Bool = false var isConnecting: Bool = false @@ -81,22 +75,72 @@ final class StreamViewModel { private var statisticsTimer: Timer? var twitchClient: TwitchAuthClient? var youtubeClient: YouTubeClient? - var youtubeBroadcastId: String? - var youtubeTransitionTask: Task? private var recordingState = RecordingState.shared + // MARK: - Backward Compatibility + + /// Primary platform (first selected, for single-platform code paths) + var selectedPlatform: StreamPlatform { + get { selectedPlatforms.first ?? .twitch } + set { + selectedPlatforms = [newValue] + } + } + + /// Stream key for the primary platform + var streamKey: String { + get { platformConfigs[selectedPlatform]?.streamKey ?? "" } + set { platformConfigs[selectedPlatform, default: PlatformConfig()].streamKey = newValue } + } + + /// YouTube broadcast ID (from primary or YouTube-specific config) + var youtubeBroadcastId: String? { + get { platformConfigs[.youtube]?.broadcastId } + set { platformConfigs[.youtube, default: PlatformConfig()].broadcastId = newValue } + } + + var youtubeTransitionTask: Task? { + get { platformConfigs[.youtube]?.transitionTask } + set { platformConfigs[.youtube, default: PlatformConfig()].transitionTask = newValue } + } + // MARK: - Computed Properties var canStartStreaming: Bool { - let hasValidKey = selectedPlatform == .arkavo || !streamKey.isEmpty - return hasValidKey && !isStreaming && !isConnecting && - (selectedPlatform != .custom || !customRTMPURL.isEmpty) + guard !isStreaming, !isConnecting else { return false } + // All selected platforms must have valid keys (or not require one) + for platform in selectedPlatforms { + if platform.requiresStreamKey { + let key = platformConfigs[platform]?.streamKey ?? "" + if key.isEmpty { return false } + } + if platform == .custom && customRTMPURL.isEmpty { return false } + } + return !selectedPlatforms.isEmpty } var effectiveRTMPURL: String { selectedPlatform == .custom ? customRTMPURL : selectedPlatform.rtmpURL } + /// Estimated total upload bitrate for all selected platforms + var estimatedTotalBitrate: String { + let perStream = Double(videoBitrate) + 128_000 // video + audio + let total = perStream * Double(selectedPlatforms.count) + if total < 1_000_000 { + return String(format: "%.0f Kbps", total / 1000) + } + return String(format: "%.1f Mbps", total / 1_000_000) + } + + private var videoBitrate: Int { + // Match the auto-detected bitrate from VideoEncoder + let cores = ProcessInfo.processInfo.activeProcessorCount + if cores >= 8 { return 4_500_000 } + if cores >= 4 { return 3_000_000 } + return 1_500_000 + } + var formattedBitrate: String { if bitrate < 1000 { return String(format: "%.0f bps", bitrate) @@ -128,7 +172,6 @@ final class StreamViewModel { func startStreaming() async { guard canStartStreaming else { return } - // Validate inputs before streaming if let validationError = validateInputs() { error = validationError return @@ -143,8 +186,8 @@ final class StreamViewModel { isConnecting = true do { - if selectedPlatform == .arkavo { - // Use NTDF-encrypted streaming for Arkavo + // Handle Arkavo NTDF separately (not part of simulcast) + if selectedPlatforms.contains(.arkavo) { guard let kasURL = URL(string: "https://100.arkavo.net") else { self.error = "Invalid KAS URL" isConnecting = false @@ -152,51 +195,40 @@ final class StreamViewModel { } try await session.startNTDFStreaming( kasURL: kasURL, - rtmpURL: effectiveRTMPURL, - streamKey: "live/creator" // Default stream key for Arkavo + rtmpURL: StreamPlatform.arkavo.rtmpURL, + streamKey: "live/creator" ) - } else { - // For YouTube, create a broadcast and bind it before starting RTMP - if selectedPlatform == .youtube, let ytClient = youtubeClient { + } + + // Build RTMP destinations for non-Arkavo platforms + let rtmpPlatforms = selectedPlatforms.filter { !$0.isEncrypted } + if !rtmpPlatforms.isEmpty { + // YouTube: create broadcast before RTMP + if rtmpPlatforms.contains(.youtube), let ytClient = youtubeClient { let broadcastId = try await ytClient.createAndBindBroadcast(title: title) - youtubeBroadcastId = broadcastId + platformConfigs[.youtube, default: PlatformConfig()].broadcastId = broadcastId debugLog("[StreamViewModel] Created YouTube broadcast: \(broadcastId)") } - // Create RTMP destination for other platforms - let destination = RTMPPublisher.Destination( - url: effectiveRTMPURL, - platform: selectedPlatform.rawValue.lowercased() - ) + var destinations: [(id: String, destination: RTMPPublisher.Destination, streamKey: String)] = [] + for platform in rtmpPlatforms { + let config = platformConfigs[platform] ?? PlatformConfig() + let url = platform == .custom ? customRTMPURL : platform.rtmpURL + let dest = RTMPPublisher.Destination(url: url, platform: platform.rawValue.lowercased()) + var key = config.streamKey + if platform == .twitch && isBandwidthTest { + key += "?bandwidthtest=true" + } + destinations.append((id: platform.rawValue.lowercased(), destination: dest, streamKey: key)) + } - // Connect and start streaming - // Append bandwidth test flag if enabled (Twitch-specific) - let effectiveStreamKey = isBandwidthTest ? "\(streamKey)?bandwidthtest=true" : streamKey - try await session.startStreaming(to: destination, streamKey: effectiveStreamKey) + try await session.startStreaming(destinations: destinations) } isStreaming = true isConnecting = false - - // Start statistics polling startStatisticsTimer() - // For YouTube with enableAutoStart, the broadcast transitions automatically - // when YouTube detects the RTMP stream. If not using autoStart, transition manually: - if selectedPlatform == .youtube, let ytClient = youtubeClient, let broadcastId = youtubeBroadcastId { - Task { - // Wait for YouTube to receive and process the RTMP data - try? await Task.sleep(nanoseconds: 10_000_000_000) // 10 seconds - do { - try await ytClient.transitionBroadcastToLive(broadcastId: broadcastId) - debugLog("[StreamViewModel] YouTube broadcast is now LIVE") - } catch { - // enableAutoStart should handle this, log but don't fail - debugLog("[StreamViewModel] YouTube transition note: \(error.localizedDescription)") - } - } - } - } catch { self.error = error.localizedDescription isConnecting = false @@ -207,12 +239,12 @@ final class StreamViewModel { func stopStreaming() async { guard let session = recordingState.getRecordingSession(), isStreaming else { return } - // Cancel YouTube transition task first, then end broadcast - youtubeTransitionTask?.cancel() - youtubeTransitionTask = nil - if let ytClient = youtubeClient, let broadcastId = youtubeBroadcastId { + // Cancel YouTube transition task and end broadcast + platformConfigs[.youtube]?.transitionTask?.cancel() + platformConfigs[.youtube]?.transitionTask = nil + if let ytClient = youtubeClient, let broadcastId = platformConfigs[.youtube]?.broadcastId { try? await ytClient.endBroadcast(broadcastId: broadcastId) - youtubeBroadcastId = nil + platformConfigs[.youtube]?.broadcastId = nil debugLog("[StreamViewModel] Ended YouTube broadcast") } @@ -220,19 +252,20 @@ final class StreamViewModel { isStreaming = false isConnecting = false - - // Stop statistics polling stopStatisticsTimer() - - // Reset statistics bitrate = 0 fps = 0 framesSent = 0 bytesSent = 0 duration = 0 + + // Clear per-platform live state + for platform in platformConfigs.keys { + platformConfigs[platform]?.isLive = false + platformConfigs[platform]?.error = nil + } } - /// Start polling stream statistics (duration, bitrate, etc.) func startStatisticsPolling() { startStatisticsTimer() } @@ -264,7 +297,6 @@ final class StreamViewModel { bytesSent = stats.bytesSent duration = stats.duration - // Calculate FPS from frames sent over duration if duration > 0 { fps = Double(framesSent) / duration } @@ -273,49 +305,47 @@ final class StreamViewModel { // MARK: - Stream Key Management func loadStreamKey() { - // Clear current key before loading platform-specific key - streamKey = "" - - // Load stream key from Keychain for selected platform - if let savedKey = KeychainManager.getStreamKey(for: selectedPlatform.rawValue) { - // Validate it's not a URL (bad cached value) - if !savedKey.hasPrefix("http://") && !savedKey.hasPrefix("https://") { - streamKey = savedKey - debugLog("[StreamViewModel] Loaded stream key for \(selectedPlatform.rawValue)") - } else { - // Clear invalid cached URL - debugLog("[StreamViewModel] Clearing invalid cached stream key (was URL)") - KeychainManager.deleteStreamKey(for: selectedPlatform.rawValue) + // Load keys for all selected platforms + for platform in selectedPlatforms { + var config = platformConfigs[platform] ?? PlatformConfig() + config.streamKey = "" + if let savedKey = KeychainManager.getStreamKey(for: platform.rawValue) { + if !savedKey.hasPrefix("http://") && !savedKey.hasPrefix("https://") { + config.streamKey = savedKey + debugLog("[StreamViewModel] Loaded stream key for \(platform.rawValue)") + } else { + KeychainManager.deleteStreamKey(for: platform.rawValue) + } } + platformConfigs[platform] = config } - // Handle custom RTMP URL - if selectedPlatform == .custom { - customRTMPURL = "" - if let savedURL = KeychainManager.getCustomRTMPURL() { - customRTMPURL = savedURL - } + if selectedPlatforms.contains(.custom) { + customRTMPURL = KeychainManager.getCustomRTMPURL() ?? "" } } func saveStreamKey() { - // Save stream key to Keychain (but never save URLs) - if !streamKey.isEmpty && !streamKey.hasPrefix("http://") && !streamKey.hasPrefix("https://") { - try? KeychainManager.saveStreamKey(streamKey, for: selectedPlatform.rawValue) - debugLog("[StreamViewModel] Saved stream key for \(selectedPlatform.rawValue)") + for platform in selectedPlatforms { + let key = platformConfigs[platform]?.streamKey ?? "" + if !key.isEmpty && !key.hasPrefix("http://") && !key.hasPrefix("https://") { + try? KeychainManager.saveStreamKey(key, for: platform.rawValue) + debugLog("[StreamViewModel] Saved stream key for \(platform.rawValue)") + } } - // Save custom RTMP URL if custom platform - if selectedPlatform == .custom && !customRTMPURL.isEmpty { + if selectedPlatforms.contains(.custom) && !customRTMPURL.isEmpty { try? KeychainManager.saveCustomRTMPURL(customRTMPURL) } } func clearStreamKey() { - KeychainManager.deleteStreamKey(for: selectedPlatform.rawValue) - streamKey = "" + for platform in selectedPlatforms { + KeychainManager.deleteStreamKey(for: platform.rawValue) + platformConfigs[platform]?.streamKey = "" + } - if selectedPlatform == .custom { + if selectedPlatforms.contains(.custom) { KeychainManager.deleteCustomRTMPURL() customRTMPURL = "" } @@ -323,111 +353,54 @@ final class StreamViewModel { // MARK: - Input Validation - /// Validates stream key, RTMP URL, and title - /// - Returns: Error message if validation fails, nil if all inputs are valid private func validateInputs() -> String? { - // Validate stream key - if let error = validateStreamKey(streamKey) { - return error - } - - // Validate custom RTMP URL if custom platform - if selectedPlatform == .custom { - if let error = validateRTMPURL(customRTMPURL) { - return error + for platform in selectedPlatforms { + if platform.requiresStreamKey { + let key = platformConfigs[platform]?.streamKey ?? "" + if let error = validateStreamKey(key, platform: platform) { + return "[\(platform.rawValue)] \(error)" + } + } + if platform == .custom { + if let error = validateRTMPURL(customRTMPURL) { + return error + } } } - // Validate stream title if let error = validateTitle(title) { return error } - return nil } - /// Validates stream key format and length - private func validateStreamKey(_ key: String) -> String? { - // Arkavo doesn't require a stream key - if selectedPlatform == .arkavo { - return nil - } - - // Check if empty - if key.trimmingCharacters(in: .whitespaces).isEmpty { - return "Stream key cannot be empty" - } - - // Check minimum length (most platforms require at least 10 characters) - if key.count < 10 { - return "Stream key is too short (minimum 10 characters)" - } - - // Check maximum length (reasonable limit for stream keys) - if key.count > 200 { - return "Stream key is too long (maximum 200 characters)" - } - - // Check for valid characters (alphanumeric, hyphens, underscores) - let validCharacterSet = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "-_")) - if key.rangeOfCharacter(from: validCharacterSet.inverted) != nil { - return "Stream key contains invalid characters (only letters, numbers, hyphens, and underscores allowed)" + private func validateStreamKey(_ key: String, platform: StreamPlatform) -> String? { + if platform == .arkavo { return nil } + if key.trimmingCharacters(in: .whitespaces).isEmpty { return "Stream key cannot be empty" } + if key.count < 10 { return "Stream key is too short (minimum 10 characters)" } + if key.count > 200 { return "Stream key is too long (maximum 200 characters)" } + let validChars = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "-_")) + if key.rangeOfCharacter(from: validChars.inverted) != nil { + return "Stream key contains invalid characters" } - return nil } - /// Validates RTMP URL format and protocol private func validateRTMPURL(_ urlString: String) -> String? { - // Check if empty - if urlString.trimmingCharacters(in: .whitespaces).isEmpty { - return "RTMP URL cannot be empty" - } - - // Check if valid URL - guard let url = URL(string: urlString) else { - return "Invalid RTMP URL format" - } - - // Check protocol - guard let scheme = url.scheme?.lowercased() else { - return "RTMP URL must specify a protocol (rtmp:// or rtmps://)" - } - - guard scheme == "rtmp" || scheme == "rtmps" else { - return "RTMP URL must use rtmp:// or rtmps:// protocol" - } - - // Check host - guard let host = url.host, !host.isEmpty else { - return "RTMP URL must include a valid host" + if urlString.trimmingCharacters(in: .whitespaces).isEmpty { return "RTMP URL cannot be empty" } + guard let url = URL(string: urlString) else { return "Invalid RTMP URL format" } + guard let scheme = url.scheme?.lowercased(), scheme == "rtmp" || scheme == "rtmps" else { + return "RTMP URL must use rtmp:// or rtmps://" } - - // Check overall length - if urlString.count > 500 { - return "RTMP URL is too long (maximum 500 characters)" - } - + guard let host = url.host, !host.isEmpty else { return "RTMP URL must include a host" } + if urlString.count > 500 { return "RTMP URL is too long" } return nil } - /// Validates stream title length and characters private func validateTitle(_ title: String) -> String? { - // Allow empty title (optional field) - if title.isEmpty { - return nil - } - - // Check maximum length - if title.count > 200 { - return "Stream title is too long (maximum 200 characters)" - } - - // Check for control characters - if title.rangeOfCharacter(from: .controlCharacters) != nil { - return "Stream title contains invalid control characters" - } - + if title.isEmpty { return nil } + if title.count > 200 { return "Stream title is too long (maximum 200 characters)" } + if title.rangeOfCharacter(from: .controlCharacters) != nil { return "Stream title contains invalid characters" } return nil } } diff --git a/ArkavoCreator/ArkavoCreator/Streaming/ChatPanelViewModel.swift b/ArkavoCreator/ArkavoCreator/Streaming/ChatPanelViewModel.swift index 3babae21..0137b29c 100644 --- a/ArkavoCreator/ArkavoCreator/Streaming/ChatPanelViewModel.swift +++ b/ArkavoCreator/ArkavoCreator/Streaming/ChatPanelViewModel.swift @@ -1,22 +1,31 @@ import Foundation +import ArkavoKit @MainActor @Observable final class ChatPanelViewModel { var messages: [ChatMessage] = [] var recentEvents: [StreamEvent] = [] - var isConnected: Bool = false + var connectedPlatforms: Set = [] var error: String? - private var chatClient: TwitchChatClient? - private var eventSubClient: TwitchEventSubClient? - private var listenerTask: Task? - private var eventListenerTask: Task? + var isConnected: Bool { !connectedPlatforms.isEmpty } + + // Twitch state + private var twitchChatClient: TwitchChatClient? + private var twitchEventSubClient: TwitchEventSubClient? + private var twitchListenerTask: Task? + private var twitchEventListenerTask: Task? + + // YouTube state + private var youtubePollingTask: Task? private static let maxMessages = 200 private static let maxEvents = 50 - func connect(twitchClient: TwitchAuthClient) { + // MARK: - Twitch + + func connectTwitch(twitchClient: TwitchAuthClient) { guard twitchClient.isAuthenticated, let token = twitchClient.accessToken, let channel = twitchClient.username else { @@ -29,13 +38,13 @@ final class ChatPanelViewModel { client.oauthToken = token client.channel = channel client.username = twitchClient.username - chatClient = client + twitchChatClient = client - listenerTask = Task { + twitchListenerTask = Task { do { try await client.connect() - isConnected = true - error = nil + connectedPlatforms.insert("twitch") + debugLog("[ChatPanel] Twitch chat connected") for await message in client.chatMessages { messages.append(message) @@ -43,23 +52,25 @@ final class ChatPanelViewModel { messages.removeFirst(messages.count - Self.maxMessages) } } - // Stream ended - isConnected = false + connectedPlatforms.remove("twitch") } catch { - self.error = error.localizedDescription - isConnected = false + self.error = "Twitch chat: \(error.localizedDescription)" + connectedPlatforms.remove("twitch") } } - // Connect EventSub for follows, subs, raids, cheers + // Connect EventSub let eventSub = TwitchEventSubClient( clientId: twitchClient.clientId, accessToken: { [weak twitchClient] in twitchClient?.accessToken }, - userId: { [weak twitchClient] in twitchClient?.userId } + userId: { [weak twitchClient] in twitchClient?.userId }, + ensureValidToken: { [weak twitchClient] in + await twitchClient?.ensureValidToken() ?? false + } ) - eventSubClient = eventSub + twitchEventSubClient = eventSub - eventListenerTask = Task { + twitchEventListenerTask = Task { await eventSub.connect() for await event in eventSub.events { @@ -71,17 +82,128 @@ final class ChatPanelViewModel { } } - func disconnect() { - listenerTask?.cancel() - listenerTask = nil - eventListenerTask?.cancel() - eventListenerTask = nil - Task { - await chatClient?.disconnect() + // MARK: - YouTube + + func connectYouTube(youtubeClient: YouTubeClient, broadcastId: String) { + youtubePollingTask = Task { + do { + guard let liveChatId = try await youtubeClient.getLiveChatId(broadcastId: broadcastId) else { + error = "No live chat available for this broadcast" + return + } + + connectedPlatforms.insert("youtube") + debugLog("[ChatPanel] YouTube chat connected (chatId: \(liveChatId))") + + var nextPageToken: String? = nil + var pollingInterval: TimeInterval = 6.0 + + while !Task.isCancelled { + do { + let result = try await youtubeClient.fetchLiveChatMessages( + liveChatId: liveChatId, + pageToken: nextPageToken + ) + nextPageToken = result.nextPageToken + + if let ms = result.pollingIntervalMs { + pollingInterval = max(Double(ms) / 1000.0, 5.0) + } + + for item in result.messages { + let author = item.authorDetails + var badges: [String] = [] + if author.isChatOwner { badges.append("owner") } + if author.isChatModerator { badges.append("moderator") } + if author.isChatSponsor { badges.append("member") } + + let chatMsg = ChatMessage( + id: item.id, + platform: "youtube", + username: author.channelId, + displayName: author.displayName, + content: item.snippet.displayMessage, + badges: badges, + isHighlighted: item.snippet.type == "superChatEvent" + ) + messages.append(chatMsg) + if messages.count > Self.maxMessages { + messages.removeFirst(messages.count - Self.maxMessages) + } + + // Super Chat → donation event + if item.snippet.type == "superChatEvent", + let details = item.snippet.superChatDetails { + let amount = (Double(details.amountMicros) ?? 0) / 1_000_000.0 + let event = StreamEvent( + platform: "youtube", + type: .donation, + username: author.channelId, + displayName: author.displayName, + message: details.userComment, + amount: amount + ) + recentEvents.append(event) + if recentEvents.count > Self.maxEvents { + recentEvents.removeFirst(recentEvents.count - Self.maxEvents) + } + } + + if item.snippet.type == "newSponsorEvent" { + let event = StreamEvent( + platform: "youtube", + type: .subscribe, + username: author.channelId, + displayName: author.displayName + ) + recentEvents.append(event) + if recentEvents.count > Self.maxEvents { + recentEvents.removeFirst(recentEvents.count - Self.maxEvents) + } + } + } + } catch { + debugLog("[ChatPanel] YouTube chat poll error: \(error.localizedDescription)") + } + + try? await Task.sleep(for: .seconds(pollingInterval)) + } + } catch { + self.error = "YouTube chat: \(error.localizedDescription)" + } + connectedPlatforms.remove("youtube") + } + } + + // MARK: - Backward Compat + + func connect(twitchClient: TwitchAuthClient) { + connectTwitch(twitchClient: twitchClient) + } + + func connect(youtubeClient: YouTubeClient, broadcastId: String) { + connectYouTube(youtubeClient: youtubeClient, broadcastId: broadcastId) + } + + // MARK: - Disconnect + + func disconnect(platform: String? = nil) { + if platform == nil || platform == "twitch" { + twitchListenerTask?.cancel() + twitchListenerTask = nil + twitchEventListenerTask?.cancel() + twitchEventListenerTask = nil + Task { await twitchChatClient?.disconnect() } + twitchChatClient = nil + twitchEventSubClient?.disconnect() + twitchEventSubClient = nil + connectedPlatforms.remove("twitch") + } + + if platform == nil || platform == "youtube" { + youtubePollingTask?.cancel() + youtubePollingTask = nil + connectedPlatforms.remove("youtube") } - chatClient = nil - eventSubClient?.disconnect() - eventSubClient = nil - isConnected = false } } diff --git a/ArkavoCreator/ArkavoCreator/Streaming/TwitchEventSubClient.swift b/ArkavoCreator/ArkavoCreator/Streaming/TwitchEventSubClient.swift new file mode 100644 index 00000000..0f0f605c --- /dev/null +++ b/ArkavoCreator/ArkavoCreator/Streaming/TwitchEventSubClient.swift @@ -0,0 +1,424 @@ +// +// TwitchEventSubClient.swift +// ArkavoCreator +// +// Twitch EventSub WebSocket client for real-time channel events. +// Receives follows, subscriptions, cheers, raids, and gift subs +// via wss://eventsub.wss.twitch.tv/ws and yields them as StreamEvents. +// + +import Foundation +import OSLog + +/// Twitch EventSub WebSocket client +@MainActor +final class TwitchEventSubClient { + private let logger = Logger(subsystem: "com.arkavo.creator", category: "TwitchEventSub") + + private var webSocket: URLSessionWebSocketTask? + private var urlSession: URLSession? + private var sessionId: String? + private var keepaliveTimeoutSeconds: Int = 30 + private var keepaliveTimer: Task? + private var receiveTask: Task? + + private var eventContinuation: AsyncStream.Continuation? + private(set) var events: AsyncStream! + + /// OAuth token and client ID for Helix API subscription calls + private let accessToken: () -> String? + private let clientId: String + private let userId: () -> String? + /// Called before subscription creation to ensure the token is valid + private let ensureValidToken: () async -> Bool + + private(set) var isConnected = false + + /// Event types to subscribe to once the session is established + private let subscriptionTypes: [(type: String, version: String, scope: String?)] = [ + ("channel.follow", "2", "moderator:read:followers"), + ("channel.subscribe", "1", "channel:read:subscriptions"), + ("channel.subscription.gift", "1", "channel:read:subscriptions"), + ("channel.cheer", "1", "bits:read"), + ("channel.raid", "1", nil), + ] + + init(clientId: String, accessToken: @escaping () -> String?, userId: @escaping () -> String?, ensureValidToken: @escaping () async -> Bool = { true }) { + self.clientId = clientId + self.accessToken = accessToken + self.userId = userId + self.ensureValidToken = ensureValidToken + + self.events = AsyncStream { continuation in + self.eventContinuation = continuation + } + } + + // MARK: - Connection + + func connect() async { + guard !isConnected else { return } + + let url = URL(string: "wss://eventsub.wss.twitch.tv/ws")! + let session = URLSession(configuration: .default) + self.urlSession = session + let ws = session.webSocketTask(with: url) + self.webSocket = ws + ws.resume() + + isConnected = true + logger.info("Connecting to Twitch EventSub WebSocket") + + receiveTask = Task { [weak self] in + await self?.receiveLoop() + } + } + + func disconnect() { + isConnected = false + keepaliveTimer?.cancel() + keepaliveTimer = nil + receiveTask?.cancel() + receiveTask = nil + webSocket?.cancel(with: .normalClosure, reason: nil) + webSocket = nil + urlSession?.invalidateAndCancel() + urlSession = nil + sessionId = nil + eventContinuation?.finish() + logger.info("Disconnected from Twitch EventSub") + } + + // MARK: - Receive Loop + + private func receiveLoop() async { + guard let ws = webSocket else { return } + + while isConnected, !Task.isCancelled { + do { + let message = try await ws.receive() + switch message { + case .string(let text): + handleMessage(text) + case .data(let data): + if let text = String(data: data, encoding: .utf8) { + handleMessage(text) + } + @unknown default: + break + } + } catch { + if isConnected { + logger.error("EventSub receive error: \(error.localizedDescription)") + isConnected = false + // Attempt reconnect after a delay + Task { [weak self] in + try? await Task.sleep(for: .seconds(5)) + await self?.reconnect() + } + } + break + } + } + } + + private func handleMessage(_ text: String) { + guard let data = text.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let metadata = json["metadata"] as? [String: Any], + let messageType = metadata["message_type"] as? String + else { + logger.warning("Failed to parse EventSub message") + return + } + + switch messageType { + case "session_welcome": + handleWelcome(json) + case "session_keepalive": + resetKeepaliveTimer() + case "notification": + handleNotification(json) + case "session_reconnect": + handleReconnect(json) + case "revocation": + if let payload = json["payload"] as? [String: Any], + let subscription = payload["subscription"] as? [String: Any], + let type = subscription["type"] as? String { + logger.warning("Subscription revoked: \(type)") + } + default: + logger.debug("Unknown EventSub message type: \(messageType)") + } + } + + // MARK: - Message Handlers + + private func handleWelcome(_ json: [String: Any]) { + guard let payload = json["payload"] as? [String: Any], + let session = payload["session"] as? [String: Any], + let id = session["id"] as? String + else { return } + + sessionId = id + if let timeout = session["keepalive_timeout_seconds"] as? Int { + keepaliveTimeoutSeconds = timeout + } + + logger.info("EventSub session established: \(id)") + resetKeepaliveTimer() + + // Subscribe to all event types + Task { [weak self] in + await self?.createSubscriptions() + } + } + + private func handleNotification(_ json: [String: Any]) { + guard let metadata = json["metadata"] as? [String: Any], + let subscriptionType = metadata["subscription_type"] as? String, + let payload = json["payload"] as? [String: Any], + let eventData = payload["event"] as? [String: Any] + else { return } + + guard let event = parseEvent(type: subscriptionType, data: eventData) else { return } + eventContinuation?.yield(event) + } + + private func handleReconnect(_ json: [String: Any]) { + guard let payload = json["payload"] as? [String: Any], + let session = payload["session"] as? [String: Any], + let reconnectURL = session["reconnect_url"] as? String + else { return } + + logger.info("EventSub reconnect requested") + Task { [weak self] in + await self?.reconnectTo(urlString: reconnectURL) + } + } + + // MARK: - Event Parsing + + private func parseEvent(type: String, data: [String: Any]) -> StreamEvent? { + switch type { + case "channel.follow": + guard let userName = data["user_login"] as? String, + let displayName = data["user_name"] as? String + else { return nil } + return StreamEvent( + platform: "twitch", + type: .follow, + username: userName, + displayName: displayName + ) + + case "channel.subscribe": + let userName = data["user_login"] as? String ?? "" + let displayName = data["user_name"] as? String ?? userName + let tier = data["tier"] as? String + let tierAmount: Double? = switch tier { + case "1000": 4.99 + case "2000": 9.99 + case "3000": 24.99 + default: nil + } + return StreamEvent( + platform: "twitch", + type: .subscribe, + username: userName, + displayName: displayName, + amount: tierAmount + ) + + case "channel.subscription.gift": + let userName = data["user_login"] as? String ?? "" + let displayName = data["user_name"] as? String ?? userName + let total = data["total"] as? Int ?? 1 + let tier = data["tier"] as? String + let perSub: Double = switch tier { + case "2000": 9.99 + case "3000": 24.99 + default: 4.99 + } + return StreamEvent( + platform: "twitch", + type: .giftSub, + username: userName, + displayName: displayName, + message: "\(total) gift sub(s)", + amount: perSub * Double(total) + ) + + case "channel.cheer": + let userName = data["user_login"] as? String ?? "Anonymous" + let displayName = data["user_name"] as? String ?? userName + let bits = data["bits"] as? Int ?? 0 + let message = data["message"] as? String + return StreamEvent( + platform: "twitch", + type: .cheer, + username: userName, + displayName: displayName, + message: message, + amount: Double(bits) + ) + + case "channel.raid": + let userName = data["from_broadcaster_user_login"] as? String ?? "" + let displayName = data["from_broadcaster_user_name"] as? String ?? userName + let viewers = data["viewers"] as? Int ?? 0 + return StreamEvent( + platform: "twitch", + type: .raid, + username: userName, + displayName: displayName, + message: "\(viewers) viewers", + amount: Double(viewers) + ) + + default: + return nil + } + } + + // MARK: - Subscriptions + + private func createSubscriptions() async { + // Validate / refresh the token before attempting subscriptions + let tokenValid = await ensureValidToken() + guard tokenValid, + let token = accessToken(), + let broadcasterId = userId(), + let sessionId + else { + logger.error("Cannot create subscriptions: missing or invalid token, userId, or sessionId") + return + } + + for sub in subscriptionTypes { + do { + try await createSubscription( + type: sub.type, + version: sub.version, + broadcasterId: broadcasterId, + token: token, + sessionId: sessionId + ) + } catch { + logger.error("Failed to subscribe to \(sub.type): \(error.localizedDescription)") + } + } + } + + private func createSubscription( + type: String, + version: String, + broadcasterId: String, + token: String, + sessionId: String + ) async throws { + var request = URLRequest(url: URL(string: "https://api.twitch.tv/helix/eventsub/subscriptions")!) + request.httpMethod = "POST" + request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") + request.setValue(clientId, forHTTPHeaderField: "Client-Id") + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + + // Build condition — most events use broadcaster_user_id, + // channel.follow v2 also needs moderator_user_id, + // channel.raid uses to_broadcaster_user_id for incoming raids + var condition: [String: String] = [:] + if type == "channel.raid" { + condition["to_broadcaster_user_id"] = broadcasterId + } else { + condition["broadcaster_user_id"] = broadcasterId + } + if type == "channel.follow" { + condition["moderator_user_id"] = broadcasterId + } + + let body: [String: Any] = [ + "type": type, + "version": version, + "condition": condition, + "transport": [ + "method": "websocket", + "session_id": sessionId, + ], + ] + + request.httpBody = try JSONSerialization.data(withJSONObject: body) + + let (data, response) = try await URLSession.shared.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse else { + throw TwitchError.apiFailed + } + + if httpResponse.statusCode == 202 { + logger.info("Subscribed to \(type)") + } else { + let responseBody = String(data: data, encoding: .utf8) ?? "no body" + logger.error("Subscribe to \(type) failed (\(httpResponse.statusCode)): \(responseBody)") + } + } + + // MARK: - Keepalive & Reconnect + + private func resetKeepaliveTimer() { + keepaliveTimer?.cancel() + keepaliveTimer = Task { [weak self, keepaliveTimeoutSeconds] in + // Twitch says connection is dead if no message within keepalive_timeout + 10s + let timeout = keepaliveTimeoutSeconds + 10 + try? await Task.sleep(for: .seconds(timeout)) + guard !Task.isCancelled else { return } + await self?.handleKeepaliveTimeout() + } + } + + private func handleKeepaliveTimeout() { + logger.warning("EventSub keepalive timeout — reconnecting") + Task { [weak self] in + await self?.reconnect() + } + } + + private func reconnect() async { + disconnect() + + // Re-create the event stream for new consumers + self.events = AsyncStream { continuation in + self.eventContinuation = continuation + } + + try? await Task.sleep(for: .seconds(1)) + await connect() + } + + private func reconnectTo(urlString: String) async { + guard let url = URL(string: urlString) else { + await reconnect() + return + } + + // Keep old connection alive until new one sends welcome + let oldWs = webSocket + let oldSession = urlSession + + let session = URLSession(configuration: .default) + self.urlSession = session + let ws = session.webSocketTask(with: url) + self.webSocket = ws + ws.resume() + + // The new connection will send a session_welcome with the same session ID + // Old connection can be closed after welcome + receiveTask?.cancel() + receiveTask = Task { [weak self] in + await self?.receiveLoop() + } + + // Clean up old connection + oldWs?.cancel(with: .normalClosure, reason: nil) + oldSession?.invalidateAndCancel() + } +} diff --git a/ArkavoCreator/ArkavoCreator/TwitchAuthClient.swift b/ArkavoCreator/ArkavoCreator/TwitchAuthClient.swift index ca31c3cc..e1bd674d 100644 --- a/ArkavoCreator/ArkavoCreator/TwitchAuthClient.swift +++ b/ArkavoCreator/ArkavoCreator/TwitchAuthClient.swift @@ -36,12 +36,13 @@ class TwitchAuthClient: ObservableObject { // MARK: - Private Properties private(set) var accessToken: String? + private var refreshToken: String? private var cancellables = Set() private var notificationObserver: NSObjectProtocol? private var authSession: ASWebAuthenticationSession? // OAuth Configuration - private let clientId: String + let clientId: String private let clientSecret: String private var redirectURI: String { ArkavoConfiguration.shared.oauthRedirectURL(for: "twitch") } private let authURL = "https://id.twitch.tv/oauth2/authorize" @@ -50,7 +51,10 @@ class TwitchAuthClient: ObservableObject { "user:read:email", "channel:read:stream_key", // Note: This scope may not actually work - Twitch restricts stream key access "channel:manage:broadcast", // Required for updating stream title, category, tags - "chat:read" // Read chat messages for Muse avatar reactions + "chat:read", // Read chat messages for Muse avatar reactions + "moderator:read:followers", // EventSub: channel.follow v2 + "channel:read:subscriptions", // EventSub: subscribe & gift sub events + "bits:read", // EventSub: cheer events ] // MARK: - Initialization @@ -184,6 +188,7 @@ class TwitchAuthClient: ObservableObject { func logout() { isAuthenticated = false accessToken = nil + refreshToken = nil username = nil userId = nil followerCount = nil @@ -486,6 +491,75 @@ class TwitchAuthClient: ObservableObject { } } + /// Validates the current token with Twitch and refreshes if expired. + /// Returns true if a valid token is available after the call. + @discardableResult + func ensureValidToken() async -> Bool { + guard let token = accessToken else { return false } + + // Validate with Twitch + var request = URLRequest(url: URL(string: "https://id.twitch.tv/oauth2/validate")!) + request.setValue("OAuth \(token)", forHTTPHeaderField: "Authorization") + + do { + let (_, response) = try await URLSession.shared.data(for: request) + if let http = response as? HTTPURLResponse, http.statusCode == 200 { + return true + } + } catch { + debugLog("❌ Token validation request failed: \(error.localizedDescription)") + } + + // Token invalid — try refresh + debugLog("🔄 Access token invalid, attempting refresh") + return await refreshAccessToken() + } + + /// Uses the stored refresh token to obtain a new access token. + private func refreshAccessToken() async -> Bool { + guard let refresh = refreshToken else { + debugLog("❌ No refresh token available — user must re-authenticate") + clearStoredCredentials() + isAuthenticated = false + return false + } + + var bodyComponents = URLComponents() + bodyComponents.queryItems = [ + URLQueryItem(name: "client_id", value: clientId), + URLQueryItem(name: "client_secret", value: clientSecret), + URLQueryItem(name: "grant_type", value: "refresh_token"), + URLQueryItem(name: "refresh_token", value: refresh), + ] + + var request = URLRequest(url: URL(string: tokenURL)!) + request.httpMethod = "POST" + request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type") + request.httpBody = bodyComponents.query?.data(using: .utf8) + + do { + let (data, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + debugLog("❌ Token refresh failed — user must re-authenticate") + clearStoredCredentials() + isAuthenticated = false + return false + } + + let tokenResponse = try JSONDecoder().decode(TwitchTokenResponse.self, from: data) + self.accessToken = tokenResponse.access_token + self.refreshToken = tokenResponse.refresh_token ?? self.refreshToken + saveStoredCredentials(token: tokenResponse.access_token, refreshToken: self.refreshToken) + debugLog("✅ Token refreshed successfully") + return true + } catch { + debugLog("❌ Token refresh error: \(error.localizedDescription)") + clearStoredCredentials() + isAuthenticated = false + return false + } + } + // MARK: - Private Methods private func exchangeCodeForToken(_ code: String) async throws { @@ -525,9 +599,10 @@ class TwitchAuthClient: ObservableObject { let tokenResponse = try JSONDecoder().decode(TwitchTokenResponse.self, from: data) self.accessToken = tokenResponse.access_token + self.refreshToken = tokenResponse.refresh_token - // Save token - saveStoredCredentials(token: tokenResponse.access_token) + // Save tokens + saveStoredCredentials(token: tokenResponse.access_token, refreshToken: tokenResponse.refresh_token) // Fetch user info try await fetchUserInfo() @@ -537,41 +612,53 @@ class TwitchAuthClient: ObservableObject { // MARK: - Keychain Storage - private func saveStoredCredentials(token: String) { - // Migrate from UserDefaults to Keychain for better security + private func saveStoredCredentials(token: String, refreshToken: String? = nil) { KeychainManager.save(value: token, service: "com.arkavo.twitch", account: "access_token") + if let refreshToken { + KeychainManager.save(value: refreshToken, service: "com.arkavo.twitch", account: "refresh_token") + } // Clean up old UserDefaults storage if it exists UserDefaults.standard.removeObject(forKey: "twitch_access_token") } private func loadStoredCredentials() { + // Load refresh token from Keychain + if let refreshData = try? KeychainManager.load(service: "com.arkavo.twitch", account: "refresh_token"), + let refresh = String(data: refreshData, encoding: .utf8) { + self.refreshToken = refresh + } + // Try Keychain first (new method) if let tokenData = try? KeychainManager.load(service: "com.arkavo.twitch", account: "access_token"), let token = String(data: tokenData, encoding: .utf8) { self.accessToken = token Task { - do { - try await fetchUserInfo() - isAuthenticated = true - } catch { - // Token might be expired - clearStoredCredentials() + // Validate token before trusting it; refresh if expired + let valid = await ensureValidToken() + if valid { + do { + try await fetchUserInfo() + isAuthenticated = true + } catch { + clearStoredCredentials() + } } } } // Fallback to UserDefaults for existing users (migration path) else if let token = UserDefaults.standard.string(forKey: "twitch_access_token") { self.accessToken = token - // Migrate to Keychain saveStoredCredentials(token: token) Task { - do { - try await fetchUserInfo() - isAuthenticated = true - } catch { - // Token might be expired - clearStoredCredentials() + let valid = await ensureValidToken() + if valid { + do { + try await fetchUserInfo() + isAuthenticated = true + } catch { + clearStoredCredentials() + } } } } @@ -579,7 +666,9 @@ class TwitchAuthClient: ObservableObject { private func clearStoredCredentials() { try? KeychainManager.delete(service: "com.arkavo.twitch", account: "access_token") + try? KeychainManager.delete(service: "com.arkavo.twitch", account: "refresh_token") UserDefaults.standard.removeObject(forKey: "twitch_access_token") + self.refreshToken = nil } } diff --git a/ArkavoKit/Sources/ArkavoRecorder/RecordingSession.swift b/ArkavoKit/Sources/ArkavoRecorder/RecordingSession.swift index 6a0f65d5..9ec7aa43 100644 --- a/ArkavoKit/Sources/ArkavoRecorder/RecordingSession.swift +++ b/ArkavoKit/Sources/ArkavoRecorder/RecordingSession.swift @@ -707,11 +707,19 @@ public final class RecordingSession: Sendable { // MARK: - Streaming - /// Start streaming to RTMP destination + /// Start streaming to a single RTMP destination public func startStreaming(to destination: RTMPPublisher.Destination, streamKey: String) async throws { try await encoder.startStreaming(to: destination, streamKey: streamKey) _streamingActive.withLock { $0 = true } - // Start frame generation if not already recording + if !_isRecording { + startStreamingFrameGeneration() + } + } + + /// Start streaming to multiple RTMP destinations simultaneously (simulcast) + public func startStreaming(destinations: [(id: String, destination: RTMPPublisher.Destination, streamKey: String)]) async throws { + try await encoder.startStreaming(to: destinations) + _streamingActive.withLock { $0 = true } if !_isRecording { startStreamingFrameGeneration() } @@ -731,13 +739,22 @@ public final class RecordingSession: Sendable { } } - /// Stop streaming + /// Stop streaming to all destinations public func stopStreaming() async { _streamingActive.withLock { $0 = false } await encoder.stopStreaming() await encoder.stopNTDFStreaming() } + /// Stop streaming to a single destination (others continue) + public func stopStreaming(id: String) async { + await encoder.stopStreaming(id: id) + // If no destinations remain, deactivate streaming + if await encoder.activeDestinationIds.isEmpty { + _streamingActive.withLock { $0 = false } + } + } + /// Get streaming statistics public var streamStatistics: RTMPPublisher.StreamStatistics? { get async { diff --git a/ArkavoKit/Sources/ArkavoRecorder/VideoEncoder.swift b/ArkavoKit/Sources/ArkavoRecorder/VideoEncoder.swift index ab6e4ca1..6affb9d4 100644 --- a/ArkavoKit/Sources/ArkavoRecorder/VideoEncoder.swift +++ b/ArkavoKit/Sources/ArkavoRecorder/VideoEncoder.swift @@ -32,22 +32,30 @@ public actor VideoEncoder { public private(set) var isRecording: Bool = false - // Streaming support - private var rtmpPublisher: RTMPPublisher? + // Streaming support — per-destination state for simulcast fan-out + struct StreamDestination { + let id: String + let publisher: RTMPPublisher + var videoSendTask: Task? + var audioSendTask: Task? + var videoContinuation: AsyncStream.Continuation? + var audioContinuation: AsyncStream.Continuation? + var sentVideoSequenceHeader: Bool = false + var sentAudioSequenceHeader: Bool = false + } + private var streamDestinations: [String: StreamDestination] = [:] private var ntdfStreamingManager: NTDFStreamingManager? - private var isStreaming: Bool = false private var isNTDFStreaming: Bool = false - /// Whether any streaming (regular RTMP or NTDF) is active — used by RecordingSession - /// to gate frame generation when streaming without recording + /// Whether any streaming (regular RTMP or NTDF) is active + private var isStreaming: Bool { !streamDestinations.isEmpty } public var isStreamingActive: Bool { isStreaming || isNTDFStreaming } + private var videoFormatDescription: CMFormatDescription? private var audioFormatDescription: CMFormatDescription? - private var sentVideoSequenceHeader: Bool = false - private var sentAudioSequenceHeader: Bool = false - private var streamStartTime: CMTime? // Stream start time for relative timestamps - private var lastStreamVideoTimestamp: CMTime = .zero // Last video timestamp sent to stream - private var lastStreamAudioTimestamp: CMTime = .zero // Last audio timestamp sent to stream + private var streamStartTime: CMTime? + private var lastStreamVideoTimestamp: CMTime = .zero + private var lastStreamAudioTimestamp: CMTime = .zero // Encoding settings - adaptive based on system capabilities private let videoWidth: Int @@ -621,181 +629,217 @@ public actor VideoEncoder { // MARK: - Streaming Methods - // Frame queue continuations for serialized sending - private var videoFrameContinuation: AsyncStream.Continuation? - private var audioFrameContinuation: AsyncStream.Continuation? - private var videoSendTask: Task? - private var audioSendTask: Task? + // Shared frame queue (not per-destination — destinations get their own via fan-out) private var silentAudioTask: Task? - /// Start streaming to RTMP destination(s) while recording + /// Start streaming to one RTMP destination (convenience wrapper) public func startStreaming(to destination: RTMPPublisher.Destination, streamKey: String) async throws { - guard !isStreaming else { + try await startStreaming(to: [(id: destination.platform, destination: destination, streamKey: streamKey)]) + } + + /// Start streaming to multiple RTMP destinations simultaneously (simulcast) + public func startStreaming(to destinations: [(id: String, destination: RTMPPublisher.Destination, streamKey: String)]) async throws { + guard streamDestinations.isEmpty else { print("⚠️ Already streaming") return } - print("📡 Starting RTMP stream...") - - let publisher = RTMPPublisher() - try await publisher.connect(to: destination, streamKey: streamKey) - - // Send stream metadata (@setDataFrame onMetaData) immediately after connect - // sendMetadata/FLVMuxer expect values in bits/sec and convert to kbps internally - try await publisher.sendMetadata( - width: videoWidth, - height: videoHeight, - framerate: Double(frameRate), - videoBitrate: Double(videoBitrate), - audioBitrate: 128_000 - ) + print("📡 Starting RTMP stream to \(destinations.count) destination(s)...") - // Create video encoder + // Create shared media encoders (encode once, fan out to all destinations) let videoEncoder = ArkavoMedia.VideoEncoder(quality: .auto) try videoEncoder.start() - - // Create audio encoder let audioEncoder = try ArkavoMedia.AudioEncoder(bitrate: 128_000) - // Create AsyncStreams to serialize frame sending (prevents burst/out-of-order issues) - let (videoStream, videoContinuation) = AsyncStream.makeStream() - let (audioStream, audioContinuation) = AsyncStream.makeStream() - self.videoFrameContinuation = videoContinuation - self.audioFrameContinuation = audioContinuation - - // Wire up video encoder callback - just queue frames - // Capture continuation locally to avoid actor isolation issues - let videoCont = videoContinuation - videoEncoder.onFrame = { frame in - videoCont.yield(frame) - } + // Connect all destinations in parallel + try await withThrowingTaskGroup(of: StreamDestination.self) { group in + for dest in destinations { + group.addTask { + let publisher = RTMPPublisher() + try await publisher.connect(to: dest.destination, streamKey: dest.streamKey) + try await publisher.sendMetadata( + width: self.videoWidth, + height: self.videoHeight, + framerate: Double(self.frameRate), + videoBitrate: Double(self.videoBitrate), + audioBitrate: 128_000 + ) + print("✅ [\(dest.id)] RTMP connected") + return StreamDestination(id: dest.id, publisher: publisher) + } + } - // Wire up audio encoder callback - just queue frames - let audioCont = audioContinuation - audioEncoder.onFrame = { frame in - audioCont.yield(frame) + for try await dest in group { + streamDestinations[dest.id] = dest + } } - // Start video send task - serializes frame sending - // Frames arrive from camera at realtime pace, so we just need to send them in order - // without additional pacing (the camera/encoder already gates the frame rate) - videoSendTask = Task { [weak self, weak publisher] in - for await frame in videoStream { - guard let self = self, let publisher = publisher else { break } - guard !Task.isCancelled else { break } + // Create per-destination AsyncStreams and send tasks + for id in streamDestinations.keys { + guard var dest = streamDestinations[id] else { continue } - do { - // Send sequence header ONLY ONCE on first keyframe - let needsHeader = await self.shouldSendVideoSequenceHeader() - if frame.isKeyframe, needsHeader, let formatDesc = frame.formatDescription { - try await publisher.sendVideoSequenceHeader(formatDescription: formatDesc) - await self.markVideoSequenceHeaderSent() - print("✅ Sent video sequence header (ONCE)") + let (videoStream, videoCont) = AsyncStream.makeStream( + bufferingPolicy: .bufferingNewest(30) + ) + let (audioStream, audioCont) = AsyncStream.makeStream( + bufferingPolicy: .bufferingNewest(30) + ) + dest.videoContinuation = videoCont + dest.audioContinuation = audioCont + + // Per-destination video send task with frame rate limiting + let publisher = dest.publisher + let destId = id + let targetInterval: Double = 1.0 / Double(self.frameRate) // ~33ms for 30fps + dest.videoSendTask = Task { [weak self] in + var lastSendTime: ContinuousClock.Instant? = nil + var frameCount: UInt64 = 0 + for await frame in videoStream { + guard let self = self else { break } + guard !Task.isCancelled else { break } + + // Rate limit: skip frames that arrive faster than target fps + let now = ContinuousClock.now + if let last = lastSendTime { + let elapsed = now - last + if elapsed < .milliseconds(Int(targetInterval * 900)) && !frame.isKeyframe { + continue // Drop frame — too fast + } } - // Send video frame immediately - frames arrive at realtime from camera - try await publisher.send(video: frame) - } catch is CancellationError { - break - } catch { - print("❌ Failed to send video frame: \(error)") + do { + let needsHeader = await self.shouldSendVideoHeader(for: destId) + if frame.isKeyframe, needsHeader, let formatDesc = frame.formatDescription { + try await publisher.sendVideoSequenceHeader(formatDescription: formatDesc) + await self.markVideoHeaderSent(for: destId) + print("✅ [\(destId)] Sent video sequence header") + } + try await publisher.send(video: frame) + lastSendTime = now + frameCount += 1 + if frameCount == 1 || frameCount % 900 == 0 { + print("📤 [\(destId)] video #\(frameCount)") + } + } catch is CancellationError { + break + } catch { + print("❌ [\(destId)] Video send error: \(error.localizedDescription)") + } } } - } - // Start audio send task - serializes frame sending - // Audio frames arrive from encoder at realtime pace - audioSendTask = Task { [weak self, weak publisher] in - for await frame in audioStream { - guard let self = self, let publisher = publisher else { break } - guard !Task.isCancelled else { break } - - do { - // Send sequence header ONLY ONCE on first frame - let needsHeader = await self.shouldSendAudioSequenceHeader() - if needsHeader, let formatDesc = frame.formatDescription { - // Extract AudioSpecificConfig from format description - var asc = Data() - var size: Int = 0 - if let cookie = CMAudioFormatDescriptionGetMagicCookie(formatDesc, sizeOut: &size), size > 0 { - asc = Data(bytes: cookie, count: size) - } else { - // Manual ASC construction for AAC-LC 48kHz stereo - let byte1: UInt8 = 0x11 // (2<<3)|(3>>1) = AAC-LC, 48kHz - let byte2: UInt8 = 0x90 // ((3&1)<<7)|(2<<3) = 48kHz, stereo - asc = Data([byte1, byte2]) + // Per-destination audio send task + dest.audioSendTask = Task { [weak self] in + var audioFrameCount: UInt64 = 0 + for await frame in audioStream { + guard let self = self else { break } + guard !Task.isCancelled else { break } + audioFrameCount += 1 + if audioFrameCount == 1 || audioFrameCount % 500 == 0 { + print("🔊 [\(destId)] audio #\(audioFrameCount) (\(frame.data.count)B)") + } + do { + let needsHeader = await self.shouldSendAudioHeader(for: destId) + if needsHeader, let formatDesc = frame.formatDescription { + var asc = Data() + var size: Int = 0 + if let cookie = CMAudioFormatDescriptionGetMagicCookie(formatDesc, sizeOut: &size), size > 0 { + asc = Data(bytes: cookie, count: size) + } else { + let byte1: UInt8 = 0x11 + let byte2: UInt8 = 0x90 + asc = Data([byte1, byte2]) + } + try await publisher.sendAudioSequenceHeader(asc: asc) + await self.markAudioHeaderSent(for: destId) + print("✅ [\(destId)] Sent audio sequence header") } - - try await publisher.sendAudioSequenceHeader(asc: asc) - await self.markAudioSequenceHeaderSent() - print("✅ Sent audio sequence header (ONCE)") + try await publisher.send(audio: frame) + } catch is CancellationError { + break + } catch { + print("❌ [\(destId)] Audio send error: \(error.localizedDescription)") } - - // Send audio frame immediately - frames arrive at realtime from encoder - try await publisher.send(audio: frame) - } catch is CancellationError { - break - } catch { - print("❌ Failed to send audio frame: \(error)") } } + + streamDestinations[id] = dest + } + + // Capture all continuations locally for the fan-out closures + // (onFrame is nonisolated, can't access actor-isolated streamDestinations) + let videoConts = streamDestinations.values.compactMap { $0.videoContinuation } + let audioConts = streamDestinations.values.compactMap { $0.audioContinuation } + + videoEncoder.onFrame = { frame in + for cont in videoConts { cont.yield(frame) } + } + audioEncoder.onFrame = { frame in + for cont in audioConts { cont.yield(frame) } } streamVideoEncoder = videoEncoder streamAudioEncoder = audioEncoder - rtmpPublisher = publisher - isStreaming = true - sentVideoSequenceHeader = false - sentAudioSequenceHeader = false streamStartTime = startTime ?? CMClockGetTime(CMClockGetHostTimeClock()) lastStreamVideoTimestamp = .zero lastStreamAudioTimestamp = .zero - // Start silent audio generator to ensure audio track is always present - // (YouTube requires audio+video to mark a stream as active) startSilentAudioGenerator(encoder: audioEncoder) - print("✅ RTMP stream started with video and audio encoding") + print("✅ RTMP stream started to \(streamDestinations.count) destination(s)") } - /// Stop streaming + /// Stop streaming to all destinations public func stopStreaming() async { - guard isStreaming, let publisher = rtmpPublisher else { return } + guard isStreaming else { return } - print("📡 Stopping RTMP stream...") + print("📡 Stopping RTMP stream (\(streamDestinations.count) destination(s))...") - // Stop silent audio generator silentAudioTask?.cancel() silentAudioTask = nil - // Finish the frame queues first - videoFrameContinuation?.finish() - audioFrameContinuation?.finish() - videoFrameContinuation = nil - audioFrameContinuation = nil - - // Wait for send tasks to complete - videoSendTask?.cancel() - audioSendTask?.cancel() - videoSendTask = nil - audioSendTask = nil - - await publisher.disconnect() + // Tear down all destinations + for (id, dest) in streamDestinations { + dest.videoContinuation?.finish() + dest.audioContinuation?.finish() + dest.videoSendTask?.cancel() + dest.audioSendTask?.cancel() + await dest.publisher.disconnect() + print("📡 [\(id)] Disconnected") + } + streamDestinations.removeAll() - // Stop encoders streamVideoEncoder?.stop() streamAudioEncoder = nil streamVideoEncoder = nil - - rtmpPublisher = nil - isStreaming = false - sentVideoSequenceHeader = false - sentAudioSequenceHeader = false streamStartTime = nil print("✅ RTMP stream stopped") } + /// Stop streaming to a single destination (others continue) + public func stopStreaming(id: String) async { + guard var dest = streamDestinations.removeValue(forKey: id) else { return } + + dest.videoContinuation?.finish() + dest.audioContinuation?.finish() + dest.videoSendTask?.cancel() + dest.audioSendTask?.cancel() + await dest.publisher.disconnect() + print("📡 [\(id)] Disconnected (remaining: \(streamDestinations.count))") + + // If no destinations left, clean up shared state + if streamDestinations.isEmpty { + silentAudioTask?.cancel() + silentAudioTask = nil + streamVideoEncoder?.stop() + streamAudioEncoder = nil + streamVideoEncoder = nil + streamStartTime = nil + print("✅ All RTMP streams stopped") + } + } + /// Generates silent PCM audio and feeds it to the audio encoder. /// Ensures the RTMP stream always has an audio track (required by YouTube). /// Real audio from mic/mixer will supplement this; the silent frames @@ -816,8 +860,8 @@ public actor VideoEncoder { while !Task.isCancelled { guard let self = self, await self.isStreaming else { break } - // Only generate silent audio if no real audio is flowing - if await !self.sentAudioSequenceHeader || true { + // Always generate silent audio as fallback + if true { // Create a CMSampleBuffer with silent PCM data var formatDesc: CMAudioFormatDescription? var asbd = AudioStreamBasicDescription( @@ -924,40 +968,32 @@ public actor VideoEncoder { // Create audio encoder let audioEncoder = try ArkavoMedia.AudioEncoder(bitrate: audioBitrate) - // Create AsyncStreams to serialize frame sending - let (videoStream, videoContinuation) = AsyncStream.makeStream() - let (audioStream, audioContinuation) = AsyncStream.makeStream() - self.videoFrameContinuation = videoContinuation - self.audioFrameContinuation = audioContinuation + // NTDF uses its own dedicated frame queues (not part of simulcast fan-out) + let (videoStream, ntdfVideoCont) = AsyncStream.makeStream() + let (audioStream, ntdfAudioCont) = AsyncStream.makeStream() + ntdfVideoContinuation = ntdfVideoCont + ntdfAudioContinuation = ntdfAudioCont - // Wire up video encoder callback - let videoCont = videoContinuation videoEncoder.onFrame = { frame in - videoCont.yield(frame) + ntdfVideoCont.yield(frame) } - - // Wire up audio encoder callback - let audioCont = audioContinuation audioEncoder.onFrame = { frame in - audioCont.yield(frame) + ntdfAudioCont.yield(frame) } - // Start video send task with encryption - videoSendTask = Task { [weak self, weak manager] in + var ntdfSentVideoHeader = false + var ntdfSentAudioHeader = false + + ntdfVideoSendTask = Task { [weak manager] in for await frame in videoStream { - guard let self = self, let manager = manager else { break } + guard let manager = manager else { break } guard !Task.isCancelled else { break } - do { - // Send sequence header ONLY ONCE on first keyframe (unencrypted) - let needsHeader = await self.shouldSendVideoSequenceHeader() - if frame.isKeyframe, needsHeader, let formatDesc = frame.formatDescription { + if frame.isKeyframe, !ntdfSentVideoHeader, let formatDesc = frame.formatDescription { try await manager.sendVideoSequenceHeader(formatDescription: formatDesc) - await self.markVideoSequenceHeaderSent() + ntdfSentVideoHeader = true print("✅ Sent video sequence header (ONCE)") } - - // Send encrypted video frame try await manager.sendEncryptedVideo(frame: frame) } catch is CancellationError { break @@ -967,32 +1003,23 @@ public actor VideoEncoder { } } - // Start audio send task with encryption - audioSendTask = Task { [weak self, weak manager] in + ntdfAudioSendTask = Task { [weak manager] in for await frame in audioStream { - guard let self = self, let manager = manager else { break } + guard let manager = manager else { break } guard !Task.isCancelled else { break } - do { - // Send sequence header ONLY ONCE on first frame (unencrypted) - let needsHeader = await self.shouldSendAudioSequenceHeader() - if needsHeader, let formatDesc = frame.formatDescription { + if !ntdfSentAudioHeader, let formatDesc = frame.formatDescription { var asc = Data() var size: Int = 0 if let cookie = CMAudioFormatDescriptionGetMagicCookie(formatDesc, sizeOut: &size), size > 0 { asc = Data(bytes: cookie, count: size) } else { - let byte1: UInt8 = 0x11 - let byte2: UInt8 = 0x90 - asc = Data([byte1, byte2]) + asc = Data([0x11, 0x90]) } - try await manager.sendAudioSequenceHeader(asc: asc) - await self.markAudioSequenceHeaderSent() + ntdfSentAudioHeader = true print("✅ Sent audio sequence header (ONCE)") } - - // Send encrypted audio frame try await manager.sendEncryptedAudio(frame: frame) } catch is CancellationError { break @@ -1006,8 +1033,6 @@ public actor VideoEncoder { streamAudioEncoder = audioEncoder ntdfStreamingManager = manager isNTDFStreaming = true - sentVideoSequenceHeader = false - sentAudioSequenceHeader = false streamStartTime = startTime ?? CMClockGetTime(CMClockGetHostTimeClock()) lastStreamVideoTimestamp = .zero lastStreamAudioTimestamp = .zero @@ -1015,35 +1040,36 @@ public actor VideoEncoder { print("✅ NTDF-encrypted stream started") } + // NTDF-specific frame queue state + private var ntdfVideoContinuation: AsyncStream.Continuation? + private var ntdfAudioContinuation: AsyncStream.Continuation? + private var ntdfVideoSendTask: Task? + private var ntdfAudioSendTask: Task? + /// Stop NTDF streaming public func stopNTDFStreaming() async { guard isNTDFStreaming, let manager = ntdfStreamingManager else { return } print("🔐 Stopping NTDF stream...") - // Finish the frame queues first - videoFrameContinuation?.finish() - audioFrameContinuation?.finish() - videoFrameContinuation = nil - audioFrameContinuation = nil + ntdfVideoContinuation?.finish() + ntdfAudioContinuation?.finish() + ntdfVideoContinuation = nil + ntdfAudioContinuation = nil - // Wait for send tasks to complete - videoSendTask?.cancel() - audioSendTask?.cancel() - videoSendTask = nil - audioSendTask = nil + ntdfVideoSendTask?.cancel() + ntdfAudioSendTask?.cancel() + ntdfVideoSendTask = nil + ntdfAudioSendTask = nil await manager.disconnect() - // Stop encoders streamVideoEncoder?.stop() streamAudioEncoder = nil streamVideoEncoder = nil ntdfStreamingManager = nil isNTDFStreaming = false - sentVideoSequenceHeader = false - sentAudioSequenceHeader = false streamStartTime = nil print("✅ NTDF stream stopped") @@ -1052,8 +1078,9 @@ public actor VideoEncoder { /// Get streaming statistics public var streamStatistics: RTMPPublisher.StreamStatistics? { get async { - if let publisher = rtmpPublisher { - return await publisher.statistics + // Return stats from first active destination + if let firstDest = streamDestinations.values.first { + return await firstDest.publisher.statistics } if let manager = ntdfStreamingManager { return await manager.statistics @@ -1062,26 +1089,33 @@ public actor VideoEncoder { } } - // MARK: - Sequence Header State Helpers (for actor-safe callback access) + /// Get statistics for a specific destination + public func streamStatistics(for id: String) async -> RTMPPublisher.StreamStatistics? { + guard let dest = streamDestinations[id] else { return nil } + return await dest.publisher.statistics + } + + /// IDs of all active streaming destinations + public var activeDestinationIds: [String] { + Array(streamDestinations.keys) + } + + // MARK: - Per-Destination Sequence Header State - /// Returns true if video sequence header has not yet been sent - private func shouldSendVideoSequenceHeader() -> Bool { - !sentVideoSequenceHeader + private func shouldSendVideoHeader(for id: String) -> Bool { + !(streamDestinations[id]?.sentVideoSequenceHeader ?? true) } - /// Marks video sequence header as sent - private func markVideoSequenceHeaderSent() { - sentVideoSequenceHeader = true + private func markVideoHeaderSent(for id: String) { + streamDestinations[id]?.sentVideoSequenceHeader = true } - /// Returns true if audio sequence header has not yet been sent - private func shouldSendAudioSequenceHeader() -> Bool { - !sentAudioSequenceHeader + private func shouldSendAudioHeader(for id: String) -> Bool { + !(streamDestinations[id]?.sentAudioSequenceHeader ?? true) } - /// Marks audio sequence header as sent - private func markAudioSequenceHeaderSent() { - sentAudioSequenceHeader = true + private func markAudioHeaderSent(for id: String) { + streamDestinations[id]?.sentAudioSequenceHeader = true } // MARK: - VTCompressionSession Setup diff --git a/ArkavoKit/Sources/ArkavoSocial/YouTubeClient.swift b/ArkavoKit/Sources/ArkavoSocial/YouTubeClient.swift index 2d0b2198..13e7d0cd 100644 --- a/ArkavoKit/Sources/ArkavoSocial/YouTubeClient.swift +++ b/ArkavoKit/Sources/ArkavoSocial/YouTubeClient.swift @@ -670,6 +670,49 @@ public actor YouTubeClient: ObservableObject { } } + /// Fetches the liveChatId for a broadcast + public func getLiveChatId(broadcastId: String) async throws -> String? { + let url = URL(string: "https://www.googleapis.com/youtube/v3/liveBroadcasts?id=\(broadcastId)&part=snippet")! + let request = try await makeAuthorizedRequest(url: url) + let (data, response) = try await URLSession.shared.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 200 else { + return nil + } + + struct BroadcastListResponse: Codable { + let items: [YouTubeBroadcastResponse] + } + let listResponse = try JSONDecoder().decode(BroadcastListResponse.self, from: data) + return listResponse.items.first?.snippet?.liveChatId + } + + /// Fetches live chat messages using OAuth token (not API key) + public func fetchLiveChatMessages(liveChatId: String, pageToken: String?) async throws -> YouTubeLiveChatResult { + var urlComponents = URLComponents(string: "https://www.googleapis.com/youtube/v3/liveChat/messages")! + urlComponents.queryItems = [ + URLQueryItem(name: "liveChatId", value: liveChatId), + URLQueryItem(name: "part", value: "snippet,authorDetails"), + ] + if let pageToken = pageToken { + urlComponents.queryItems?.append(URLQueryItem(name: "pageToken", value: pageToken)) + } + + let request = try await makeAuthorizedRequest(url: urlComponents.url!) + let (data, response) = try await URLSession.shared.data(for: request) + + guard let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 200 else { + throw YouTubeError.httpError(statusCode: (response as? HTTPURLResponse)?.statusCode ?? 0) + } + + let chatResponse = try JSONDecoder().decode(YouTubeLiveChatResponse.self, from: data) + return YouTubeLiveChatResult( + messages: chatResponse.items, + nextPageToken: chatResponse.nextPageToken, + pollingIntervalMs: chatResponse.pollingIntervalMillis + ) + } + /// Creates a live stream and returns its ID (not just the stream key) private func createLiveStreamAndReturnId() async throws -> String { let url = URL(string: "https://www.googleapis.com/youtube/v3/liveStreams?part=snippet,cdn,contentDetails")! @@ -702,8 +745,13 @@ public actor YouTubeClient: ObservableObject { struct YouTubeBroadcastResponse: Codable { let id: String + let snippet: Snippet? let status: Status? + struct Snippet: Codable { + let liveChatId: String? + } + struct Status: Codable { let lifeCycleStatus: String? } @@ -837,3 +885,44 @@ public enum YouTubeError: LocalizedError { } } } + +// MARK: - Live Chat Response Types + +public struct YouTubeLiveChatResult: Sendable { + public let messages: [YouTubeLiveChatMessage] + public let nextPageToken: String? + public let pollingIntervalMs: Int? +} + +public struct YouTubeLiveChatResponse: Codable { + public let nextPageToken: String? + public let pollingIntervalMillis: Int? + public let items: [YouTubeLiveChatMessage] +} + +public struct YouTubeLiveChatMessage: Codable, Sendable { + public let id: String + public let snippet: Snippet + public let authorDetails: AuthorDetails + + public struct Snippet: Codable, Sendable { + public let type: String + public let displayMessage: String + public let publishedAt: String + public let superChatDetails: SuperChatDetails? + + public struct SuperChatDetails: Codable, Sendable { + public let amountMicros: String + public let currency: String + public let userComment: String? + } + } + + public struct AuthorDetails: Codable, Sendable { + public let channelId: String + public let displayName: String + public let isChatOwner: Bool + public let isChatModerator: Bool + public let isChatSponsor: Bool + } +} From 7488be5284702b3c801f3acdbe79d14e2a3add3c Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Sat, 4 Apr 2026 21:15:11 -0400 Subject: [PATCH 09/14] Upgrade to mlx-swift-lm with Gemma 4 support - Replace mlx-swift-examples 2.29.1 with mlx-swift 0.31.3 + mlx-swift-lm (arkavo-ai fork with Gemma 4 text model at 73.7 tok/s) - Add MLXHuggingFace, Tokenizers, HuggingFace dependencies - Update MLXBackend to use #huggingFaceLoadModelContainer macro - Add Gemma 4 E4B (8B params, 4B active MoE, 8-bit) to ModelRegistry - Fix Sendable capture in MLXBackend.generate() Co-Authored-By: Claude Opus 4.6 (1M context) --- .../xcshareddata/swiftpm/Package.resolved | 135 ++++++++++++++++++ MuseCore/Package.resolved | 98 +++++++++++-- MuseCore/Package.swift | 13 +- .../Sources/MuseCore/LLM/MLXBackend.swift | 32 +++-- .../Sources/MuseCore/LLM/ModelRegistry.swift | 8 ++ 5 files changed, 259 insertions(+), 27 deletions(-) diff --git a/Arkavo.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Arkavo.xcworkspace/xcshareddata/swiftpm/Package.resolved index c96af2ee..4f4b2bcd 100644 --- a/Arkavo.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Arkavo.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -10,6 +10,15 @@ "version" : "1.9.0" } }, + { + "identity" : "eventsource", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattt/EventSource.git", + "state" : { + "revision" : "a3a85a85214caf642abaa96ae664e4c772a59f6e", + "version" : "1.4.1" + } + }, { "identity" : "flatbuffers", "kind" : "remoteSourceControl", @@ -28,6 +37,24 @@ "version" : "0.3.0" } }, + { + "identity" : "mlx-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ml-explore/mlx-swift", + "state" : { + "revision" : "61b9e011e09a62b489f6bd647958f1555bdf2896", + "version" : "0.31.3" + } + }, + { + "identity" : "mlx-swift-lm", + "kind" : "remoteSourceControl", + "location" : "https://github.com/arkavo-ai/mlx-swift-lm", + "state" : { + "branch" : "feature/gemma4-text", + "revision" : "d514e90e5962064e925c4bbc30bdd6b9afbb42e6" + } + }, { "identity" : "opentdfkit", "kind" : "remoteSourceControl", @@ -37,6 +64,105 @@ "revision" : "d8ffeff99e00ec3334aa57b1fe5f9e1c7f38d2a9" } }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "9f542610331815e29cc3821d3b6f488db8715517", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "b601256eab081c0f92f059e12818ac1d4f178ff7", + "version" : "1.3.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "6675bc0ff86e61436e615df6fc5174e043e57924", + "version" : "1.4.1" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "bb4ba815dab96d4edc1e0b86d7b9acf9ff973a84", + "version" : "4.3.1" + } + }, + { + "identity" : "swift-huggingface", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-huggingface", + "state" : { + "revision" : "b721959445b617d0bf03910b2b4aced345fd93bf", + "version" : "0.9.0" + } + }, + { + "identity" : "swift-jinja", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-jinja.git", + "state" : { + "revision" : "0aeefadec459ce8e11a333769950fb86183aca43", + "version" : "2.3.5" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "558f24a4647193b5a0e2104031b71c55d31ff83a", + "version" : "2.97.1" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "7c6ad0fc39d0763e0b699210e4124afd5041c5df", + "version" : "1.6.4" + } + }, + { + "identity" : "swift-transformers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-transformers", + "state" : { + "revision" : "b38443e44d93eca770f2eb68e2a4d0fa100f9aa2", + "version" : "1.3.0" + } + }, { "identity" : "vrmmetalkit", "kind" : "remoteSourceControl", @@ -46,6 +172,15 @@ "version" : "0.9.2" } }, + { + "identity" : "yyjson", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ibireme/yyjson.git", + "state" : { + "revision" : "8b4a38dc994a110abaec8a400615567bd996105f", + "version" : "0.12.0" + } + }, { "identity" : "zipfoundation", "kind" : "remoteSourceControl", diff --git a/MuseCore/Package.resolved b/MuseCore/Package.resolved index 6c472929..658b9b26 100644 --- a/MuseCore/Package.resolved +++ b/MuseCore/Package.resolved @@ -1,13 +1,13 @@ { - "originHash" : "47118e32930d6dfe6cf25b3c75b765c32a01978ddd3e3b9132af7f2d27fd500d", + "originHash" : "248a885a67cf1a84243209f54065f3d70e3837d2e393f4569b200d9d62783063", "pins" : [ { - "identity" : "gzipswift", + "identity" : "eventsource", "kind" : "remoteSourceControl", - "location" : "https://github.com/1024jp/GzipSwift", + "location" : "https://github.com/mattt/EventSource.git", "state" : { - "revision" : "731037f6cc2be2ec01562f6597c1d0aa3fe6fd05", - "version" : "6.0.1" + "revision" : "a3a85a85214caf642abaa96ae664e4c772a59f6e", + "version" : "1.4.1" } }, { @@ -15,17 +15,35 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/ml-explore/mlx-swift", "state" : { - "revision" : "072b684acaae80b6a463abab3a103732f33774bf", - "version" : "0.29.1" + "revision" : "61b9e011e09a62b489f6bd647958f1555bdf2896", + "version" : "0.31.3" } }, { - "identity" : "mlx-swift-examples", + "identity" : "mlx-swift-lm", "kind" : "remoteSourceControl", - "location" : "https://github.com/ml-explore/mlx-swift-examples", + "location" : "https://github.com/arkavo-ai/mlx-swift-lm", "state" : { - "revision" : "9bff95ca5f0b9e8c021acc4d71a2bbe4a7441631", - "version" : "2.29.1" + "branch" : "feature/gemma4-text", + "revision" : "376ffd0537fbc9591e2c58b377180eb58691b60a" + } + }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "9f542610331815e29cc3821d3b6f488db8715517", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "b601256eab081c0f92f059e12818ac1d4f178ff7", + "version" : "1.3.0" } }, { @@ -37,6 +55,24 @@ "version" : "1.4.0" } }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "bb4ba815dab96d4edc1e0b86d7b9acf9ff973a84", + "version" : "4.3.1" + } + }, + { + "identity" : "swift-huggingface", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-huggingface.git", + "state" : { + "revision" : "b721959445b617d0bf03910b2b4aced345fd93bf", + "version" : "0.9.0" + } + }, { "identity" : "swift-jinja", "kind" : "remoteSourceControl", @@ -46,6 +82,15 @@ "version" : "2.3.2" } }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "558f24a4647193b5a0e2104031b71c55d31ff83a", + "version" : "2.97.1" + } + }, { "identity" : "swift-numerics", "kind" : "remoteSourceControl", @@ -55,13 +100,31 @@ "version" : "1.1.1" } }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "7c6ad0fc39d0763e0b699210e4124afd5041c5df", + "version" : "1.6.4" + } + }, { "identity" : "swift-transformers", "kind" : "remoteSourceControl", "location" : "https://github.com/huggingface/swift-transformers", "state" : { - "revision" : "a2e184dddb4757bc943e77fbe99ac6786c53f0b2", - "version" : "1.0.0" + "revision" : "b38443e44d93eca770f2eb68e2a4d0fa100f9aa2", + "version" : "1.3.0" } }, { @@ -72,6 +135,15 @@ "revision" : "f84cea22aa7dc60470a2fd691f30e4c59c5d31a3", "version" : "0.9.2" } + }, + { + "identity" : "yyjson", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ibireme/yyjson.git", + "state" : { + "revision" : "8b4a38dc994a110abaec8a400615567bd996105f", + "version" : "0.12.0" + } } ], "version" : 3 diff --git a/MuseCore/Package.swift b/MuseCore/Package.swift index 419d1365..f8e0dc83 100644 --- a/MuseCore/Package.swift +++ b/MuseCore/Package.swift @@ -15,15 +15,22 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/arkavo-org/VRMMetalKit", exact: "0.9.2"), - .package(url: "https://github.com/ml-explore/mlx-swift-examples", from: "2.29.1"), + .package(url: "https://github.com/ml-explore/mlx-swift", from: "0.31.3"), + .package(url: "https://github.com/arkavo-ai/mlx-swift-lm", branch: "feature/gemma4-text"), + .package(url: "https://github.com/huggingface/swift-transformers", from: "1.2.1"), + .package(url: "https://github.com/huggingface/swift-huggingface.git", from: "0.9.0"), ], targets: [ .target( name: "MuseCore", dependencies: [ "VRMMetalKit", - .product(name: "MLXLLM", package: "mlx-swift-examples"), - .product(name: "MLXLMCommon", package: "mlx-swift-examples"), + .product(name: "MLX", package: "mlx-swift"), + .product(name: "MLXLLM", package: "mlx-swift-lm"), + .product(name: "MLXLMCommon", package: "mlx-swift-lm"), + .product(name: "MLXHuggingFace", package: "mlx-swift-lm"), + .product(name: "Tokenizers", package: "swift-transformers"), + .product(name: "HuggingFace", package: "swift-huggingface"), ], swiftSettings: [ .enableExperimentalFeature("StrictConcurrency") diff --git a/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift index 9d940ebf..0ac491ab 100644 --- a/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift +++ b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift @@ -2,10 +2,13 @@ import Foundation import MLX import MLXLMCommon import MLXLLM +import MLXHuggingFace +import HuggingFace +import Tokenizers import Synchronization /// MLX-based streaming LLM provider for on-device inference. -/// Wraps mlx-swift-examples v2 model loading and generation. +/// Uses mlx-swift-lm with HuggingFace download and tokenization. public final class MLXBackend: @unchecked Sendable { private let state = Mutex(BackendState()) @@ -19,15 +22,22 @@ public final class MLXBackend: @unchecked Sendable { } } - /// Load a model by HuggingFace ID + /// Load a model by HuggingFace ID (downloads on first use, cached after) public func loadModel(_ huggingFaceID: String, onProgress: (@Sendable (Double) -> Void)? = nil) async throws { - let container = try await MLXLMCommon.loadModelContainer( - id: huggingFaceID - ) { progress in - let fraction = progress.fractionCompleted - debugPrint("Loading \(huggingFaceID): \(Int(fraction * 100))%") - onProgress?(fraction) - } + let config = ModelConfiguration( + id: huggingFaceID, + defaultPrompt: "Hello", + extraEOSTokens: [""] + ) + + let container = try await #huggingFaceLoadModelContainer( + configuration: config, + progressHandler: { progress in + let fraction = progress.fractionCompleted + debugPrint("Loading \(huggingFaceID): \(Int(fraction * 100))%") + onProgress?(fraction) + } + ) // Set memory limit to 75% of system RAM for safety let systemMemoryGB = ProcessInfo.processInfo.physicalMemory / (1024 * 1024 * 1024) @@ -40,7 +50,7 @@ public final class MLXBackend: @unchecked Sendable { /// Unload the current model to free GPU memory public func unloadModel() { state.withLock { $0.modelContainer = nil } - MLX.GPU.clearCache() + MLX.Memory.clearCache() } public func generate( @@ -73,7 +83,7 @@ public final class MLXBackend: @unchecked Sendable { repetitionPenalty: 1.1 ) - try await container.perform { context in + try await container.perform(nonSendable: userInput) { context, userInput in let lmInput = try await context.processor.prepare(input: userInput) let stream = try MLXLMCommon.generate( input: lmInput, diff --git a/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift b/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift index e336c856..b19fbd52 100644 --- a/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift +++ b/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift @@ -46,6 +46,14 @@ public enum ModelRegistry { parameterCount: "0.8B", quantization: "bf16" ), + ModelInfo( + id: "gemma-4-e4b", + displayName: "Gemma 4 E4B", + huggingFaceID: "mlx-community/gemma-4-e4b-it-8bit", + estimatedMemoryMB: 9000, + parameterCount: "8B (4B active MoE)", + quantization: "8-bit" + ), ModelInfo( id: "qwen3.5-9b", displayName: "Qwen 3.5 9B", From 0c74903e098d5af7fe7500b0d9756ace6acc7518 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Sat, 4 Apr 2026 21:22:10 -0400 Subject: [PATCH 10/14] Remove Gemma 3 270M, make Gemma 4 E4B the default model Co-Authored-By: Claude Opus 4.6 (1M context) --- .../Sources/MuseCore/LLM/ModelRegistry.swift | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift b/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift index b19fbd52..1179e7c7 100644 --- a/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift +++ b/MuseCore/Sources/MuseCore/LLM/ModelRegistry.swift @@ -31,12 +31,12 @@ public enum ModelRegistry { /// All supported models, ordered by size public static let models: [ModelInfo] = [ ModelInfo( - id: "gemma-3-270m", - displayName: "Gemma 3 270M", - huggingFaceID: "mlx-community/gemma-3-270m-it-bf16", - estimatedMemoryMB: 906, - parameterCount: "270M", - quantization: "bf16" + id: "gemma-4-e4b", + displayName: "Gemma 4 E4B", + huggingFaceID: "mlx-community/gemma-4-e4b-it-8bit", + estimatedMemoryMB: 9000, + parameterCount: "8B (4B active MoE)", + quantization: "8-bit" ), ModelInfo( id: "qwen3.5-0.8b", @@ -46,14 +46,6 @@ public enum ModelRegistry { parameterCount: "0.8B", quantization: "bf16" ), - ModelInfo( - id: "gemma-4-e4b", - displayName: "Gemma 4 E4B", - huggingFaceID: "mlx-community/gemma-4-e4b-it-8bit", - estimatedMemoryMB: 9000, - parameterCount: "8B (4B active MoE)", - quantization: "8-bit" - ), ModelInfo( id: "qwen3.5-9b", displayName: "Qwen 3.5 9B", @@ -64,7 +56,7 @@ public enum ModelRegistry { ), ] - /// The default model (smallest, already cached) + /// The default model public static let defaultModel = models[0] /// Find a model by its ID From 1af58de67adfd9aa3df1b7cc75a54224836a0d71 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Sat, 4 Apr 2026 21:26:11 -0400 Subject: [PATCH 11/14] Use shared HF cache to reuse Python-downloaded models Sandboxed apps default to Library/Caches/huggingface/hub inside the app container. This means models downloaded by the Python CLI (at ~/.cache/huggingface/hub) aren't found, triggering a 9 GB re-download. Fix: explicitly configure HubClient with the shared cache path so the app finds models cached by any HF client. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../Sources/MuseCore/LLM/MLXBackend.swift | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift index 0ac491ab..3f7b182c 100644 --- a/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift +++ b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift @@ -22,7 +22,8 @@ public final class MLXBackend: @unchecked Sendable { } } - /// Load a model by HuggingFace ID (downloads on first use, cached after) + /// Load a model by HuggingFace ID (downloads on first use, cached after). + /// Uses shared ~/.cache/huggingface/hub to reuse models downloaded by Python CLI. public func loadModel(_ huggingFaceID: String, onProgress: (@Sendable (Double) -> Void)? = nil) async throws { let config = ModelConfiguration( id: huggingFaceID, @@ -30,14 +31,21 @@ public final class MLXBackend: @unchecked Sendable { extraEOSTokens: [""] ) - let container = try await #huggingFaceLoadModelContainer( - configuration: config, - progressHandler: { progress in - let fraction = progress.fractionCompleted - debugPrint("Loading \(huggingFaceID): \(Int(fraction * 100))%") - onProgress?(fraction) - } - ) + // Use shared HF cache (not sandboxed container) so Python-downloaded models are found + let sharedCache = HubCache(location: .init( + path: "~/.cache/huggingface/hub")) + let hub = HubClient(cache: sharedCache) + let downloader = #hubDownloader(hub) + let tokenizerLoader = #huggingFaceTokenizerLoader() + + let container = try await LLMModelFactory.shared.loadContainer( + from: downloader, using: tokenizerLoader, + configuration: config + ) { progress in + let fraction = progress.fractionCompleted + debugPrint("Loading \(huggingFaceID): \(Int(fraction * 100))%") + onProgress?(fraction) + } // Set memory limit to 75% of system RAM for safety let systemMemoryGB = ProcessInfo.processInfo.physicalMemory / (1024 * 1024 * 1024) From ec6b17e4e5a2fdd7dc787b4caa11979bf4e4255a Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Sat, 4 Apr 2026 21:33:45 -0400 Subject: [PATCH 12/14] Add AI Model settings: model picker, cache folder, debug logging Settings UI: - Preferred model picker (Gemma 4 E4B, Qwen 3.5 0.8B, Qwen 3.5 9B) - Model state indicator (idle/downloading/loading/ready/error) - Load/Unload/Retry buttons - Custom model cache folder with NSOpenPanel folder picker - Persisted via UserDefaults Debug logging: - MLXBackend: logs cache directory resolution, model cache check, load start/success - ModelManager: logs init state, auto-load decisions, load lifecycle, errors with full descriptions Co-Authored-By: Claude Opus 4.6 (1M context) --- ArkavoCreator/ArkavoCreator/ContentView.swift | 140 +++++++++++++++++- .../Sources/MuseCore/LLM/MLXBackend.swift | 30 +++- .../Sources/MuseCore/LLM/ModelManager.swift | 46 +++++- 3 files changed, 207 insertions(+), 9 deletions(-) diff --git a/ArkavoCreator/ArkavoCreator/ContentView.swift b/ArkavoCreator/ArkavoCreator/ContentView.swift index 3da8bcaf..f6ed7b2c 100644 --- a/ArkavoCreator/ArkavoCreator/ContentView.swift +++ b/ArkavoCreator/ArkavoCreator/ContentView.swift @@ -895,7 +895,7 @@ struct SectionContainer: View { .transition(.moveAndFade()) .id("assistant") case .settings: - SettingsContent(agentService: agentService) + SettingsContent(agentService: agentService, modelManager: modelManager) .transition(.moveAndFade()) .id("settings") default: @@ -1389,6 +1389,7 @@ struct SettingsContent: View { @State private var modelsPath: String = "" @State private var libraryPath: String = "" var agentService: CreatorAgentService? + var modelManager: ModelManager? var body: some View { ScrollView { @@ -1499,6 +1500,11 @@ struct SettingsContent: View { .clipShape(RoundedRectangle(cornerRadius: 10)) } + // AI Model Settings Section + if let modelManager { + AIModelSettingsSection(modelManager: modelManager) + } + // AI Agent Settings Section if FeatureFlags.aiAgent, let agentService { AgentSettingsSection(agentService: agentService) @@ -1524,6 +1530,138 @@ struct SettingsContent: View { } } +// MARK: - AI Model Settings Section + +struct AIModelSettingsSection: View { + @Bindable var modelManager: ModelManager + @State private var customCachePath: String = "" + + var body: some View { + GroupBox { + VStack(alignment: .leading, spacing: 16) { + // Preferred Model Picker + VStack(alignment: .leading, spacing: 4) { + Text("Preferred Model") + .font(.subheadline) + Picker("Model", selection: Binding( + get: { modelManager.selectedModel }, + set: { model in Task { await modelManager.selectModel(model) } } + )) { + ForEach(ModelRegistry.models) { model in + HStack { + Text(model.displayName) + Text("(\(model.quantization))") + .foregroundStyle(.secondary) + } + .tag(model) + } + } + .pickerStyle(.menu) + } + + // Model State + HStack(spacing: 8) { + switch modelManager.state { + case .idle: + Image(systemName: "circle") + .foregroundStyle(.secondary) + Text("Not loaded") + .foregroundStyle(.secondary) + case .downloading(let progress): + ProgressView(value: progress) + .frame(width: 60) + Text("Downloading \(Int(progress * 100))%") + case .loading: + ProgressView() + .controlSize(.small) + Text("Loading into memory...") + case .ready: + Image(systemName: "checkmark.circle.fill") + .foregroundStyle(.green) + Text("Ready (\(modelManager.selectedModel.parameterCount))") + case .error(let msg): + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.red) + Text(msg) + .lineLimit(2) + .font(.caption) + case .unloaded(let reason): + Image(systemName: "moon.zzz") + .foregroundStyle(.secondary) + Text("Unloaded: \(reason)") + } + Spacer() + + // Load / Unload button + if modelManager.state == .ready { + Button("Unload") { + Task { await modelManager.unloadModel() } + } + .foregroundStyle(.secondary) + } else if case .idle = modelManager.state { + Button("Load") { + Task { await modelManager.loadSelectedModel() } + } + } else if case .error = modelManager.state { + Button("Retry") { + Task { await modelManager.loadSelectedModel() } + } + } + } + .font(.subheadline) + + Divider() + + // Model Cache Folder + VStack(alignment: .leading, spacing: 4) { + HStack { + VStack(alignment: .leading, spacing: 4) { + Text("Model Cache Folder") + .font(.subheadline) + Text(customCachePath.isEmpty ? "~/.cache/huggingface/hub (default)" : customCachePath) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + } + + Spacer() + + Button("Choose...") { + let panel = NSOpenPanel() + panel.canChooseDirectories = true + panel.canChooseFiles = false + panel.allowsMultipleSelection = false + panel.message = "Select the folder containing HuggingFace model caches" + if panel.runModal() == .OK, let url = panel.url { + modelManager.customCacheDirectory = url + customCachePath = url.path + } + } + + if modelManager.customCacheDirectory != nil { + Button("Reset") { + modelManager.customCacheDirectory = nil + customCachePath = "" + } + .foregroundStyle(.secondary) + } + } + + Text("Models are downloaded from HuggingFace on first use. Reuse models cached by Python or other tools.") + .foregroundStyle(.tertiary) + .font(.caption) + } + } + } label: { + Label("AI Model", systemImage: "cpu") + } + .onAppear { + customCachePath = modelManager.customCacheDirectory?.path ?? "" + } + } +} + // MARK: - Agent Settings Section struct AgentSettingsSection: View { diff --git a/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift index 3f7b182c..e31c548e 100644 --- a/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift +++ b/MuseCore/Sources/MuseCore/LLM/MLXBackend.swift @@ -6,14 +6,19 @@ import MLXHuggingFace import HuggingFace import Tokenizers import Synchronization +import OSLog /// MLX-based streaming LLM provider for on-device inference. /// Uses mlx-swift-lm with HuggingFace download and tokenization. public final class MLXBackend: @unchecked Sendable { private let state = Mutex(BackendState()) + private let logger = Logger(subsystem: "com.arkavo.musecore", category: "MLXBackend") public let providerName = "MLX Local" + /// Custom model cache directory (nil = use shared HF cache) + public var customCacheDirectory: URL? + public init() {} public var isAvailable: Bool { @@ -23,7 +28,6 @@ public final class MLXBackend: @unchecked Sendable { } /// Load a model by HuggingFace ID (downloads on first use, cached after). - /// Uses shared ~/.cache/huggingface/hub to reuse models downloaded by Python CLI. public func loadModel(_ huggingFaceID: String, onProgress: (@Sendable (Double) -> Void)? = nil) async throws { let config = ModelConfiguration( id: huggingFaceID, @@ -31,13 +35,30 @@ public final class MLXBackend: @unchecked Sendable { extraEOSTokens: [""] ) - // Use shared HF cache (not sandboxed container) so Python-downloaded models are found - let sharedCache = HubCache(location: .init( - path: "~/.cache/huggingface/hub")) + // Determine cache location + let cacheLocation: CacheLocationProvider + if let customDir = customCacheDirectory { + logger.info("Using custom model cache: \(customDir.path)") + cacheLocation = .fixed(directory: customDir) + } else { + logger.info("Using shared HF cache: ~/.cache/huggingface/hub") + cacheLocation = .init(path: "~/.cache/huggingface/hub") + } + + let sharedCache = HubCache(location: cacheLocation) + logger.info("Cache directory resolved to: \(sharedCache.cacheDirectory.path)") + + // Check if model is already in cache + let modelDir = sharedCache.cacheDirectory + .appendingPathComponent("models--\(huggingFaceID.replacingOccurrences(of: "/", with: "--"))") + let isCached = FileManager.default.fileExists(atPath: modelDir.path) + logger.info("Model \(huggingFaceID) cached: \(isCached) at \(modelDir.path)") + let hub = HubClient(cache: sharedCache) let downloader = #hubDownloader(hub) let tokenizerLoader = #huggingFaceTokenizerLoader() + logger.info("Starting model load: \(huggingFaceID)") let container = try await LLMModelFactory.shared.loadContainer( from: downloader, using: tokenizerLoader, configuration: config @@ -46,6 +67,7 @@ public final class MLXBackend: @unchecked Sendable { debugPrint("Loading \(huggingFaceID): \(Int(fraction * 100))%") onProgress?(fraction) } + logger.info("Model loaded successfully: \(huggingFaceID)") // Set memory limit to 75% of system RAM for safety let systemMemoryGB = ProcessInfo.processInfo.physicalMemory / (1024 * 1024 * 1024) diff --git a/MuseCore/Sources/MuseCore/LLM/ModelManager.swift b/MuseCore/Sources/MuseCore/LLM/ModelManager.swift index 8d4011df..e0c2b0b8 100644 --- a/MuseCore/Sources/MuseCore/LLM/ModelManager.swift +++ b/MuseCore/Sources/MuseCore/LLM/ModelManager.swift @@ -1,5 +1,6 @@ import Foundation import Observation +import OSLog /// State of model lifecycle public enum ModelState: Equatable, Sendable { @@ -15,6 +16,8 @@ public enum ModelState: Equatable, Sendable { @Observable @MainActor public final class ModelManager { + private let logger = Logger(subsystem: "com.arkavo.musecore", category: "ModelManager") + public private(set) var state: ModelState = .idle public private(set) var selectedModel: ModelInfo = ModelRegistry.defaultModel public private(set) var availableModels: [ModelInfo] = [] @@ -27,12 +30,41 @@ public final class ModelManager { /// The MLX backend for streaming generation public var streamingProvider: MLXBackend { backend } + /// Custom model cache directory (persisted via UserDefaults) + public var customCacheDirectory: URL? { + didSet { + backend.customCacheDirectory = customCacheDirectory + if let dir = customCacheDirectory { + UserDefaults.standard.set(dir.path, forKey: "MLXModelCacheDirectory") + logger.info("Custom cache directory set: \(dir.path)") + } else { + UserDefaults.standard.removeObject(forKey: "MLXModelCacheDirectory") + logger.info("Custom cache directory cleared, using default") + } + } + } + public init() { backend = MLXBackend() + + // Restore persisted cache directory + if let savedPath = UserDefaults.standard.string(forKey: "MLXModelCacheDirectory") { + let url = URL(fileURLWithPath: savedPath) + customCacheDirectory = url + backend.customCacheDirectory = url + logger.info("Restored custom cache directory: \(savedPath)") + } + refreshAvailableModels() + logger.info("ModelManager init: \(self.availableModels.count) models available, default=\(self.selectedModel.displayName)") + logger.info("Selected model cached: \(ModelRegistry.isModelCached(self.selectedModel))") + // Auto-load the default model if it's already cached on disk - if ModelRegistry.isModelCached(selectedModel) { - Task { await loadSelectedModel() } + if ModelRegistry.isModelCached(self.selectedModel) { + logger.info("Auto-loading cached model: \(self.selectedModel.huggingFaceID)") + Task { await self.loadSelectedModel() } + } else { + logger.info("Default model not cached, skipping auto-load") } } @@ -62,6 +94,7 @@ public final class ModelManager { public func loadSelectedModel() async { switch state { case .loading, .downloading, .ready: + logger.info("loadSelectedModel: skipping, already in state \(String(describing: self.state))") return case .idle, .error, .unloaded: break @@ -70,23 +103,28 @@ public final class ModelManager { loadGeneration += 1 let currentGeneration = loadGeneration let isCached = ModelRegistry.isModelCached(selectedModel) + logger.info("loadSelectedModel: \(self.selectedModel.huggingFaceID), cached=\(isCached), generation=\(currentGeneration)") state = isCached ? .loading : .downloading(progress: 0) do { try await backend.loadModel(selectedModel.huggingFaceID) { [weak self] progress in - // Only show download progress when actually downloading from network guard !isCached else { return } Task { @MainActor in guard let self, self.loadGeneration == currentGeneration else { return } self.state = .downloading(progress: progress) } } - guard loadGeneration == currentGeneration else { return } + guard loadGeneration == currentGeneration else { + logger.warning("loadSelectedModel: generation mismatch, discarding") + return + } state = .loading await Task.yield() state = .ready + logger.info("loadSelectedModel: model ready") } catch { guard loadGeneration == currentGeneration else { return } + logger.error("loadSelectedModel: failed: \(error.localizedDescription)") state = .error(error.localizedDescription) } } From 4a875e1e5a6a7917a511be3cc6f92c94f1317f4b Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Sat, 4 Apr 2026 21:38:55 -0400 Subject: [PATCH 13/14] Update Package.resolved for mlx-swift-lm dependency Sync project-level Package.resolved with workspace-level to fix Xcode Cloud build failure from stale mlx-swift-examples reference. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../xcshareddata/swiftpm/Package.resolved | 115 +++++++++++++++--- 1 file changed, 98 insertions(+), 17 deletions(-) diff --git a/ArkavoCreator/ArkavoCreator.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/ArkavoCreator/ArkavoCreator.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 59fdbeb8..4f4b2bcd 100644 --- a/ArkavoCreator/ArkavoCreator.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/ArkavoCreator/ArkavoCreator.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "0366ce2b01e3ae8fada8f469cb2ad5bc7009430f7bb1fe2322f7c4c059171bb9", + "originHash" : "b83820b1903c2b4e25758a74513a6cf115920ddc7a5730a528e6e691a3c49c6a", "pins" : [ { "identity" : "cryptoswift", @@ -11,12 +11,21 @@ } }, { - "identity" : "gzipswift", + "identity" : "eventsource", "kind" : "remoteSourceControl", - "location" : "https://github.com/1024jp/GzipSwift", + "location" : "https://github.com/mattt/EventSource.git", "state" : { - "revision" : "731037f6cc2be2ec01562f6597c1d0aa3fe6fd05", - "version" : "6.0.1" + "revision" : "a3a85a85214caf642abaa96ae664e4c772a59f6e", + "version" : "1.4.1" + } + }, + { + "identity" : "flatbuffers", + "kind" : "remoteSourceControl", + "location" : "https://github.com/google/flatbuffers.git", + "state" : { + "revision" : "a2cd1ea3b6d3fee220106b5fed3f7ce8da9eb757", + "version" : "24.12.23" } }, { @@ -33,17 +42,17 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/ml-explore/mlx-swift", "state" : { - "revision" : "072b684acaae80b6a463abab3a103732f33774bf", - "version" : "0.29.1" + "revision" : "61b9e011e09a62b489f6bd647958f1555bdf2896", + "version" : "0.31.3" } }, { - "identity" : "mlx-swift-examples", + "identity" : "mlx-swift-lm", "kind" : "remoteSourceControl", - "location" : "https://github.com/ml-explore/mlx-swift-examples", + "location" : "https://github.com/arkavo-ai/mlx-swift-lm", "state" : { - "revision" : "9bff95ca5f0b9e8c021acc4d71a2bbe4a7441631", - "version" : "2.29.1" + "branch" : "feature/gemma4-text", + "revision" : "d514e90e5962064e925c4bbc30bdd6b9afbb42e6" } }, { @@ -55,13 +64,49 @@ "revision" : "d8ffeff99e00ec3334aa57b1fe5f9e1c7f38d2a9" } }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "9f542610331815e29cc3821d3b6f488db8715517", + "version" : "1.6.0" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "b601256eab081c0f92f059e12818ac1d4f178ff7", + "version" : "1.3.0" + } + }, { "identity" : "swift-collections", "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "8d9834a6189db730f6264db7556a7ffb751e99ee", - "version" : "1.4.0" + "revision" : "6675bc0ff86e61436e615df6fc5174e043e57924", + "version" : "1.4.1" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "bb4ba815dab96d4edc1e0b86d7b9acf9ff973a84", + "version" : "4.3.1" + } + }, + { + "identity" : "swift-huggingface", + "kind" : "remoteSourceControl", + "location" : "https://github.com/huggingface/swift-huggingface", + "state" : { + "revision" : "b721959445b617d0bf03910b2b4aced345fd93bf", + "version" : "0.9.0" } }, { @@ -69,8 +114,17 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/huggingface/swift-jinja.git", "state" : { - "revision" : "f731f03bf746481d4fda07f817c3774390c4d5b9", - "version" : "2.3.2" + "revision" : "0aeefadec459ce8e11a333769950fb86183aca43", + "version" : "2.3.5" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "558f24a4647193b5a0e2104031b71c55d31ff83a", + "version" : "2.97.1" } }, { @@ -82,13 +136,31 @@ "version" : "1.1.1" } }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-syntax.git", + "state" : { + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "7c6ad0fc39d0763e0b699210e4124afd5041c5df", + "version" : "1.6.4" + } + }, { "identity" : "swift-transformers", "kind" : "remoteSourceControl", "location" : "https://github.com/huggingface/swift-transformers", "state" : { - "revision" : "a2e184dddb4757bc943e77fbe99ac6786c53f0b2", - "version" : "1.0.0" + "revision" : "b38443e44d93eca770f2eb68e2a4d0fa100f9aa2", + "version" : "1.3.0" } }, { @@ -100,6 +172,15 @@ "version" : "0.9.2" } }, + { + "identity" : "yyjson", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ibireme/yyjson.git", + "state" : { + "revision" : "8b4a38dc994a110abaec8a400615567bd996105f", + "version" : "0.12.0" + } + }, { "identity" : "zipfoundation", "kind" : "remoteSourceControl", From 3f1a9ff6f5328515391e63f1c58ca4c002026ba4 Mon Sep 17 00:00:00 2001 From: Paul Flynn Date: Sat, 4 Apr 2026 21:40:34 -0400 Subject: [PATCH 14/14] Fix Sendable data race in RegistrationView profile creation Capture Sendable values (String) before the Task boundary instead of sending the non-Sendable Profile across isolation domains. Co-Authored-By: Claude Opus 4.6 (1M context) --- Arkavo/Arkavo/RegistrationView.swift | 30 +++++++++++++++------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/Arkavo/Arkavo/RegistrationView.swift b/Arkavo/Arkavo/RegistrationView.swift index bbec54df..ec64a4e2 100644 --- a/Arkavo/Arkavo/RegistrationView.swift +++ b/Arkavo/Arkavo/RegistrationView.swift @@ -279,25 +279,27 @@ struct RegistrationView: View { // generatedScreenNames = [] case .generateScreenName: if skipPasskeysFlag { - let newProfile = Profile( - name: selectedScreenName, - interests: Array(selectedInterests).joined(separator: ","), - hasHighEncryption: true, - hasHighIdentityAssurance: true, - ) - Task { await onComplete(newProfile) } + let name = selectedScreenName + let interests = Array(selectedInterests).joined(separator: ",") + let complete = onComplete + Task { + let profile = Profile( + name: name, interests: interests, + hasHighEncryption: true, hasHighIdentityAssurance: true) + await complete(profile) + } } else { currentStep = .enablePasskeys } case .enablePasskeys: - let newProfile = Profile( - name: selectedScreenName, - interests: Array(selectedInterests).joined(separator: ","), - hasHighEncryption: true, - hasHighIdentityAssurance: true, - ) + let name = selectedScreenName + let interests = Array(selectedInterests).joined(separator: ",") + let complete = onComplete Task { - await onComplete(newProfile) + let profile = Profile( + name: name, interests: interests, + hasHighEncryption: true, hasHighIdentityAssurance: true) + await complete(profile) } } }