Skip to content

Add voice macros. #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zachlatta merged 2 commits into from
Apr 4, 2026
+248 −19
Merged

Add voice macros. #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
20158c2
feat: add voice macros.
Mar 23, 2026
69b5224
fix: enforce exact matching for macro commands.
Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 95 additions & 19 deletions Sources/AppState.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,23 @@ import ServiceManagement
import ApplicationServices
import ScreenCaptureKit
import os.log

private let recordingLog = OSLog(subsystem: "com.zachlatta.freeflow", category: "Recording")

struct VoiceMacro: Codable, Identifiable, Equatable {
var id: UUID = UUID()
var command: String
var payload: String
}

struct PrecomputedMacro {
let original: VoiceMacro
let normalizedCommand: String
}

enum SettingsTab: String, CaseIterable, Identifiable {
case general
case prompts
case macros
case runLog

var id: String { rawValue }
Expand All @@ -21,6 +32,7 @@ enum SettingsTab: String, CaseIterable, Identifiable {
switch self {
case .general: return "General"
case .prompts: return "Prompts"
case .macros: return "Voice Macros"
case .runLog: return "Run Log"
}
}
Expand All @@ -29,6 +41,7 @@ enum SettingsTab: String, CaseIterable, Identifiable {
switch self {
case .general: return "gearshape"
case .prompts: return "text.bubble"
case .macros: return "music.mic"
case .runLog: return "clock.arrow.circlepath"
}
}
Expand Down Expand Up @@ -115,6 +128,7 @@ final class AppState: ObservableObject, @unchecked Sendable {
private let preserveClipboardStorageKey = "preserve_clipboard"
private let forceHTTP2TranscriptionStorageKey = "force_http2_transcription"
private let soundVolumeStorageKey = "sound_volume"
private let voiceMacrosStorageKey = "voice_macros"
private let transcribingIndicatorDelay: TimeInterval = 1.0
private let clipboardRestoreDelay: TimeInterval = 0.15
let maxPipelineHistoryCount = 20
Expand Down Expand Up @@ -220,6 +234,17 @@ final class AppState: ObservableObject, @unchecked Sendable {
}
}

private var precomputedMacros: [PrecomputedMacro] = []

@Published var voiceMacros: [VoiceMacro] = [] {
didSet {
if let data = try? JSONEncoder().encode(voiceMacros) {
UserDefaults.standard.set(data, forKey: voiceMacrosStorageKey)
}
precomputeMacros()
}
}

@Published var isRecording = false
@Published var isTranscribing = false
@Published var lastTranscript: String = ""
Expand Down Expand Up @@ -293,6 +318,15 @@ final class AppState: ObservableObject, @unchecked Sendable {
let forceHTTP2Transcription = UserDefaults.standard.bool(forKey: forceHTTP2TranscriptionStorageKey)
let soundVolume: Float = UserDefaults.standard.object(forKey: soundVolumeStorageKey) != nil
? UserDefaults.standard.float(forKey: soundVolumeStorageKey) : 1.0

let initialMacros: [VoiceMacro]
if let data = UserDefaults.standard.data(forKey: "voice_macros"),
let decoded = try? JSONDecoder().decode([VoiceMacro].self, from: data) {
initialMacros = decoded
} else {
initialMacros = []
}

let initialAccessibility = AXIsProcessTrusted()
let initialScreenCapturePermission = CGPreflightScreenCaptureAccess()
var removedAudioFileNames: [String] = []
Expand Down Expand Up @@ -325,11 +359,13 @@ final class AppState: ObservableObject, @unchecked Sendable {
self.preserveClipboard = preserveClipboard
self.forceHTTP2Transcription = forceHTTP2Transcription
self.soundVolume = soundVolume
self.voiceMacros = initialMacros
self.pipelineHistory = savedHistory
self.hasAccessibility = initialAccessibility
self.hasScreenRecordingPermission = initialScreenCapturePermission
self.launchAtLogin = SMAppService.mainApp.status == .enabled
self.selectedMicrophoneID = selectedMicrophoneID
self.precomputeMacros()

refreshAvailableMicrophones()
installAudioDeviceListener()
Expand Down Expand Up @@ -937,6 +973,56 @@ final class AppState: ObservableObject, @unchecked Sendable {
}
}

private func precomputeMacros() {
precomputedMacros = voiceMacros.map { macro in
PrecomputedMacro(
original: macro,
normalizedCommand: normalize(macro.command)
)
}
}

private func normalize(_ text: String) -> String {
let lowercased = text.lowercased()
let strippedPunctuation = lowercased.components(separatedBy: CharacterSet.punctuationCharacters).joined()
return strippedPunctuation.trimmingCharacters(in: .whitespacesAndNewlines)
}

private func findMatchingMacro(for transcript: String) -> VoiceMacro? {
let normalizedTranscript = normalize(transcript)
guard !normalizedTranscript.isEmpty else { return nil }

return precomputedMacros.first {
normalizedTranscript == $0.normalizedCommand
}?.original
}

private func processTranscript(
_ rawTranscript: String,
context: AppContext,
postProcessingService: PostProcessingService,
customVocabulary: String,
customSystemPrompt: String
) async -> (finalTranscript: String, status: String, prompt: String) {
if let macro = findMatchingMacro(for: rawTranscript) {
os_log(.info, log: recordingLog, "Voice macro triggered: %{public}@", macro.command)
return (macro.payload, "Voice macro used: \(macro.command)", "")
}

do {
let result = try await postProcessingService.postProcess(
transcript: rawTranscript,
context: context,
customVocabulary: customVocabulary,
customSystemPrompt: customSystemPrompt
)
return (result.transcript, "Post-processing succeeded", result.prompt)
} catch {
os_log(.error, log: recordingLog, "Post-processing failed: %{public}@", error.localizedDescription)
return (rawTranscript, "Post-processing failed, using raw transcript", "")
}
}

private func stopAndTranscribe() {
cancelPendingShortcutStart()
shortcutSessionController.reset()
Expand Down Expand Up @@ -1009,24 +1095,14 @@ final class AppState: ObservableObject, @unchecked Sendable {
await MainActor.run { [weak self] in
self?.debugStatusMessage = "Running post-processing"
}
let finalTranscript: String
let processingStatus: String
let postProcessingPrompt: String
do {
let postProcessingResult = try await postProcessingService.postProcess(
transcript: rawTranscript,
context: appContext,
customVocabulary: customVocabulary,
customSystemPrompt: customSystemPrompt
)
finalTranscript = postProcessingResult.transcript
processingStatus = "Post-processing succeeded"
postProcessingPrompt = postProcessingResult.prompt
} catch {
finalTranscript = rawTranscript
processingStatus = "Post-processing failed, using raw transcript"
postProcessingPrompt = ""
}
let (finalTranscript, processingStatus, postProcessingPrompt) = await processTranscript(
rawTranscript,
context: appContext,
postProcessingService: postProcessingService,
customVocabulary: customVocabulary,
customSystemPrompt: customSystemPrompt
)

await MainActor.run {
self.lastContextSummary = appContext.contextSummary
self.lastContextScreenshotDataURL = appContext.screenshotDataURL
Expand Down
153 changes: 153 additions & 0 deletions Sources/SettingsView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ struct SettingsView: View {
GeneralSettingsView()
case .prompts:
PromptsSettingsView()
case .macros:
VoiceMacrosSettingsView()
case .runLog:
RunLogView()
}
Expand Down Expand Up @@ -1677,3 +1679,154 @@ struct FlowLayout: Layout {
return (CGSize(width: maxWidth, height: totalHeight), positions)
}
}

// MARK: - Voice Macros Settings

struct VoiceMacrosSettingsView: View {
@EnvironmentObject var appState: AppState
@State private var showingAddMacro = false
@State private var editingMacro: VoiceMacro?

var body: some View {
ScrollView {
VStack(spacing: 20) {
SettingsCard("Voice Macros", icon: "music.mic") {
macrosSection
}
}
.padding(24)
}
.sheet(isPresented: $showingAddMacro, onDismiss: { editingMacro = nil }) {
VoiceMacroEditorView(isPresented: $showingAddMacro, macro: $editingMacro)
}
}

private var macrosSection: some View {
VStack(alignment: .leading, spacing: 12) {
HStack {
Text("Bypass post-processing and immediately paste your predefined text.")
.font(.caption)
.foregroundStyle(.secondary)
Spacer()
Button(action: { showingAddMacro = true }) {
Text("Add Macro")
}
}

if appState.voiceMacros.isEmpty {
VStack {
Image(systemName: "music.mic")
.font(.system(size: 30))
.foregroundStyle(.tertiary)
.padding(.bottom, 4)
Text("No Voice Macros Yet")
.font(.headline)
.foregroundStyle(.secondary)
Text("Click 'Add Macro' to define your first voice macro.")
.font(.caption)
.foregroundStyle(.tertiary)
.multilineTextAlignment(.center)
}
.frame(maxWidth: .infinity)
.padding(.vertical, 32)
} else {
VStack(spacing: 1) {
ForEach(Array(appState.voiceMacros.enumerated()), id: \.element.id) { index, macro in
VStack(alignment: .leading, spacing: 4) {
HStack {
Text(macro.command)
.font(.headline)
Spacer()
Button("Edit") {
editingMacro = macro
showingAddMacro = true
}
.buttonStyle(.borderless)
.font(.caption)

Button("Delete") {
appState.voiceMacros.removeAll { $0.id == macro.id }
}
.buttonStyle(.borderless)
.font(.caption)
.foregroundStyle(.red)
}
Text(macro.payload)
.font(.caption)
.foregroundStyle(.secondary)
.lineLimit(2)
}
.padding(12)
.background(Color(nsColor: .controlBackgroundColor).opacity(0.8))
}
}
.cornerRadius(8)
.overlay(RoundedRectangle(cornerRadius: 8).stroke(Color.primary.opacity(0.06), lineWidth: 1))
}
}
}
}

struct VoiceMacroEditorView: View {
@EnvironmentObject var appState: AppState
@Binding var isPresented: Bool
@Binding var macro: VoiceMacro?

@State private var command: String = ""
@State private var payload: String = ""

var body: some View {
VStack(spacing: 20) {
Text(macro == nil ? "Add Macro" : "Edit Macro")
.font(.headline)

VStack(alignment: .leading, spacing: 8) {
Text("Voice Command (What you say)")
.font(.caption.weight(.semibold))
TextField("e.g. debugging prompt", text: $command)
.textFieldStyle(.roundedBorder)

Text("Text (What gets pasted)")
.font(.caption.weight(.semibold))
.padding(.top, 8)
TextEditor(text: $payload)
.font(.system(.body, design: .monospaced))
.frame(height: 150)
.overlay(RoundedRectangle(cornerRadius: 6).stroke(Color.secondary.opacity(0.3), lineWidth: 1))
}

HStack {
Button("Cancel") {
isPresented = false
macro = nil
}
Spacer()
Button("Save") {
let newMacro = VoiceMacro(
id: macro?.id ?? UUID(),
command: command.trimmingCharacters(in: .whitespacesAndNewlines),
payload: payload
)

if let existingIndex = appState.voiceMacros.firstIndex(where: { $0.id == newMacro.id }) {
appState.voiceMacros[existingIndex] = newMacro
} else {
appState.voiceMacros.append(newMacro)
}
isPresented = false
macro = nil
}
.disabled(command.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || payload.isEmpty)
.buttonStyle(.borderedProminent)
}
}
.padding(20)
.frame(width: 400)
.onAppear {
if let m = macro {
command = m.command
payload = m.payload
}
}
}
}