From 20158c274578f3ed1ec3c74045c1f526fe434554 Mon Sep 17 00:00:00 2001 From: Kunal Jain Date: Tue, 24 Mar 2026 00:04:12 +0530 Subject: [PATCH 1/2] feat: add voice macros. --- Sources/AppState.swift | 115 +++++++++++++++++++++++----- Sources/SettingsView.swift | 153 +++++++++++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+), 19 deletions(-) diff --git a/Sources/AppState.swift b/Sources/AppState.swift index df41ba1..a02adfd 100644 --- a/Sources/AppState.swift +++ b/Sources/AppState.swift @@ -7,12 +7,23 @@ import ServiceManagement import ApplicationServices import ScreenCaptureKit import os.log - private let recordingLog = OSLog(subsystem: "com.zachlatta.freeflow", category: "Recording") +struct VoiceMacro: Codable, Identifiable, Equatable { + var id: UUID = UUID() + var command: String + var payload: String +} + +struct PrecomputedMacro { + let original: VoiceMacro + let normalizedCommand: String +} + enum SettingsTab: String, CaseIterable, Identifiable { case general case prompts + case macros case runLog var id: String { rawValue } @@ -21,6 +32,7 @@ enum SettingsTab: String, CaseIterable, Identifiable { switch self { case .general: return "General" case .prompts: return "Prompts" + case .macros: return "Voice Macros" case .runLog: return "Run Log" } } @@ -29,6 +41,7 @@ enum SettingsTab: String, CaseIterable, Identifiable { switch self { case .general: return "gearshape" case .prompts: return "text.bubble" + case .macros: return "music.mic" case .runLog: return "clock.arrow.circlepath" } } @@ -115,6 +128,7 @@ final class AppState: ObservableObject, @unchecked Sendable { private let preserveClipboardStorageKey = "preserve_clipboard" private let forceHTTP2TranscriptionStorageKey = "force_http2_transcription" private let soundVolumeStorageKey = "sound_volume" + private let voiceMacrosStorageKey = "voice_macros" private let transcribingIndicatorDelay: TimeInterval = 1.0 private let clipboardRestoreDelay: TimeInterval = 0.15 let maxPipelineHistoryCount = 20 @@ -220,6 +234,17 @@ final class AppState: ObservableObject, @unchecked Sendable { } } + private var precomputedMacros: [PrecomputedMacro] = [] + + @Published var voiceMacros: [VoiceMacro] = [] { + didSet { + if let data = try? JSONEncoder().encode(voiceMacros) { + UserDefaults.standard.set(data, forKey: voiceMacrosStorageKey) + } + precomputeMacros() + } + } + @Published var isRecording = false @Published var isTranscribing = false @Published var lastTranscript: String = "" @@ -293,6 +318,15 @@ final class AppState: ObservableObject, @unchecked Sendable { let forceHTTP2Transcription = UserDefaults.standard.bool(forKey: forceHTTP2TranscriptionStorageKey) let soundVolume: Float = UserDefaults.standard.object(forKey: soundVolumeStorageKey) != nil ? UserDefaults.standard.float(forKey: soundVolumeStorageKey) : 1.0 + + let initialMacros: [VoiceMacro] + if let data = UserDefaults.standard.data(forKey: "voice_macros"), + let decoded = try? JSONDecoder().decode([VoiceMacro].self, from: data) { + initialMacros = decoded + } else { + initialMacros = [] + } + let initialAccessibility = AXIsProcessTrusted() let initialScreenCapturePermission = CGPreflightScreenCaptureAccess() var removedAudioFileNames: [String] = [] @@ -325,11 +359,13 @@ final class AppState: ObservableObject, @unchecked Sendable { self.preserveClipboard = preserveClipboard self.forceHTTP2Transcription = forceHTTP2Transcription self.soundVolume = soundVolume + self.voiceMacros = initialMacros self.pipelineHistory = savedHistory self.hasAccessibility = initialAccessibility self.hasScreenRecordingPermission = initialScreenCapturePermission self.launchAtLogin = SMAppService.mainApp.status == .enabled self.selectedMicrophoneID = selectedMicrophoneID + self.precomputeMacros() refreshAvailableMicrophones() installAudioDeviceListener() @@ -937,6 +973,57 @@ final class AppState: ObservableObject, @unchecked Sendable { } } + private func precomputeMacros() { + precomputedMacros = voiceMacros.map { macro in + PrecomputedMacro( + original: macro, + normalizedCommand: normalize(macro.command) + ) + } + } + + private func normalize(_ text: String) -> String { + let lowercased = text.lowercased() + let strippedPunctuation = lowercased.components(separatedBy: CharacterSet.punctuationCharacters).joined() + return strippedPunctuation.trimmingCharacters(in: .whitespacesAndNewlines) + } + + private func findMatchingMacro(for transcript: String) -> VoiceMacro? { + let normalizedTranscript = normalize(transcript) + guard !normalizedTranscript.isEmpty else { return nil } + + return precomputedMacros.first { + normalizedTranscript == $0.normalizedCommand || + normalizedTranscript.contains($0.normalizedCommand) + }?.original + } + + private func processTranscript( + _ rawTranscript: String, + context: AppContext, + postProcessingService: PostProcessingService, + customVocabulary: String, + customSystemPrompt: String + ) async -> (finalTranscript: String, status: String, prompt: String) { + if let macro = findMatchingMacro(for: rawTranscript) { + os_log(.info, log: recordingLog, "Voice macro triggered: %{public}@", macro.command) + return (macro.payload, "Voice macro used: \(macro.command)", "") + } + + do { + let result = try await postProcessingService.postProcess( + transcript: rawTranscript, + context: context, + customVocabulary: customVocabulary, + customSystemPrompt: customSystemPrompt + ) + return (result.transcript, "Post-processing succeeded", result.prompt) + } catch { + os_log(.error, log: recordingLog, "Post-processing failed: %{public}@", error.localizedDescription) + return (rawTranscript, "Post-processing failed, using raw transcript", "") + } + } + private func stopAndTranscribe() { cancelPendingShortcutStart() shortcutSessionController.reset() @@ -1009,24 +1096,14 @@ final class AppState: ObservableObject, @unchecked Sendable { await MainActor.run { [weak self] in self?.debugStatusMessage = "Running post-processing" } - let finalTranscript: String - let processingStatus: String - let postProcessingPrompt: String - do { - let postProcessingResult = try await postProcessingService.postProcess( - transcript: rawTranscript, - context: appContext, - customVocabulary: customVocabulary, - customSystemPrompt: customSystemPrompt - ) - finalTranscript = postProcessingResult.transcript - processingStatus = "Post-processing succeeded" - postProcessingPrompt = postProcessingResult.prompt - } catch { - finalTranscript = rawTranscript - processingStatus = "Post-processing failed, using raw transcript" - postProcessingPrompt = "" - } + let (finalTranscript, processingStatus, postProcessingPrompt) = await processTranscript( + rawTranscript, + context: appContext, + postProcessingService: postProcessingService, + customVocabulary: customVocabulary, + customSystemPrompt: customSystemPrompt + ) + await MainActor.run { self.lastContextSummary = appContext.contextSummary self.lastContextScreenshotDataURL = appContext.screenshotDataURL diff --git a/Sources/SettingsView.swift b/Sources/SettingsView.swift index ee41761..08d146d 100644 --- a/Sources/SettingsView.swift +++ b/Sources/SettingsView.swift @@ -77,6 +77,8 @@ struct SettingsView: View { GeneralSettingsView() case .prompts: PromptsSettingsView() + case .macros: + VoiceMacrosSettingsView() case .runLog: RunLogView() } @@ -1677,3 +1679,154 @@ struct FlowLayout: Layout { return (CGSize(width: maxWidth, height: totalHeight), positions) } } + +// MARK: - Voice Macros Settings + +struct VoiceMacrosSettingsView: View { + @EnvironmentObject var appState: AppState + @State private var showingAddMacro = false + @State private var editingMacro: VoiceMacro? + + var body: some View { + ScrollView { + VStack(spacing: 20) { + SettingsCard("Voice Macros", icon: "music.mic") { + macrosSection + } + } + .padding(24) + } + .sheet(isPresented: $showingAddMacro, onDismiss: { editingMacro = nil }) { + VoiceMacroEditorView(isPresented: $showingAddMacro, macro: $editingMacro) + } + } + + private var macrosSection: some View { + VStack(alignment: .leading, spacing: 12) { + HStack { + Text("Bypass post-processing and immediately paste your predefined text.") + .font(.caption) + .foregroundStyle(.secondary) + Spacer() + Button(action: { showingAddMacro = true }) { + Text("Add Macro") + } + } + + if appState.voiceMacros.isEmpty { + VStack { + Image(systemName: "music.mic") + .font(.system(size: 30)) + .foregroundStyle(.tertiary) + .padding(.bottom, 4) + Text("No Voice Macros Yet") + .font(.headline) + .foregroundStyle(.secondary) + Text("Click 'Add Macro' to define your first voice macro.") + .font(.caption) + .foregroundStyle(.tertiary) + .multilineTextAlignment(.center) + } + .frame(maxWidth: .infinity) + .padding(.vertical, 32) + } else { + VStack(spacing: 1) { + ForEach(Array(appState.voiceMacros.enumerated()), id: \.element.id) { index, macro in + VStack(alignment: .leading, spacing: 4) { + HStack { + Text(macro.command) + .font(.headline) + Spacer() + Button("Edit") { + editingMacro = macro + showingAddMacro = true + } + .buttonStyle(.borderless) + .font(.caption) + + Button("Delete") { + appState.voiceMacros.removeAll { $0.id == macro.id } + } + .buttonStyle(.borderless) + .font(.caption) + .foregroundStyle(.red) + } + Text(macro.payload) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(2) + } + .padding(12) + .background(Color(nsColor: .controlBackgroundColor).opacity(0.8)) + } + } + .cornerRadius(8) + .overlay(RoundedRectangle(cornerRadius: 8).stroke(Color.primary.opacity(0.06), lineWidth: 1)) + } + } + } +} + +struct VoiceMacroEditorView: View { + @EnvironmentObject var appState: AppState + @Binding var isPresented: Bool + @Binding var macro: VoiceMacro? + + @State private var command: String = "" + @State private var payload: String = "" + + var body: some View { + VStack(spacing: 20) { + Text(macro == nil ? "Add Macro" : "Edit Macro") + .font(.headline) + + VStack(alignment: .leading, spacing: 8) { + Text("Voice Command (What you say)") + .font(.caption.weight(.semibold)) + TextField("e.g. debugging prompt", text: $command) + .textFieldStyle(.roundedBorder) + + Text("Text (What gets pasted)") + .font(.caption.weight(.semibold)) + .padding(.top, 8) + TextEditor(text: $payload) + .font(.system(.body, design: .monospaced)) + .frame(height: 150) + .overlay(RoundedRectangle(cornerRadius: 6).stroke(Color.secondary.opacity(0.3), lineWidth: 1)) + } + + HStack { + Button("Cancel") { + isPresented = false + macro = nil + } + Spacer() + Button("Save") { + let newMacro = VoiceMacro( + id: macro?.id ?? UUID(), + command: command.trimmingCharacters(in: .whitespacesAndNewlines), + payload: payload + ) + + if let existingIndex = appState.voiceMacros.firstIndex(where: { $0.id == newMacro.id }) { + appState.voiceMacros[existingIndex] = newMacro + } else { + appState.voiceMacros.append(newMacro) + } + isPresented = false + macro = nil + } + .disabled(command.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || payload.isEmpty) + .buttonStyle(.borderedProminent) + } + } + .padding(20) + .frame(width: 400) + .onAppear { + if let m = macro { + command = m.command + payload = m.payload + } + } + } +} From 69b52244dd95c459ad028bd13fb7b9eb83fca638 Mon Sep 17 00:00:00 2001 From: Kunal Jain Date: Fri, 3 Apr 2026 22:16:56 +0530 Subject: [PATCH 2/2] fix: enforce exact matching for macro commands. --- Sources/AppState.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Sources/AppState.swift b/Sources/AppState.swift index a02adfd..df405a0 100644 --- a/Sources/AppState.swift +++ b/Sources/AppState.swift @@ -993,8 +993,7 @@ final class AppState: ObservableObject, @unchecked Sendable { guard !normalizedTranscript.isEmpty else { return nil } return precomputedMacros.first { - normalizedTranscript == $0.normalizedCommand || - normalizedTranscript.contains($0.normalizedCommand) + normalizedTranscript == $0.normalizedCommand }?.original }