From 5beecc329f1679f0ec5926613b48c5afef428ff9 Mon Sep 17 00:00:00 2001 From: Aaron Barton Date: Tue, 7 Apr 2026 22:46:20 -0500 Subject: [PATCH] Add screenshot capture settings for token and payload optimization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose three user-configurable settings in the menu bar panel: - Primary screen only toggle (skips secondary monitors) - Active window only toggle (captures frontmost window instead of full screen) - JPEG quality slider (0.3–1.0, adjusts upload payload size) Settings persist via UserDefaults and are wired through the capture utility to both the voice response and onboarding demo pipelines. Includes unit tests for defaults, persistence, and boundary values. --- leanring-buddy/CompanionManager.swift | 47 +++- leanring-buddy/CompanionPanelView.swift | 114 ++++++++ .../CompanionScreenCaptureUtility.swift | 78 +++++- .../ScreenshotSettingsTests.swift | 255 ++++++++++++++++++ 4 files changed, 478 insertions(+), 16 deletions(-) create mode 100644 leanring-buddyTests/ScreenshotSettingsTests.swift diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 0234cf19..83db177a 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -139,6 +139,39 @@ final class CompanionManager: ObservableObject { } } + // MARK: - Screenshot Settings + + /// When enabled, only the screen containing the cursor is captured and + /// sent to Claude. Reduces token usage on multi-monitor setups. + /// Off by default so Claude has full multi-monitor context. + @Published var captureOnlyPrimaryScreen: Bool = UserDefaults.standard.bool(forKey: "captureOnlyPrimaryScreen") + + func setCaptureOnlyPrimaryScreen(_ enabled: Bool) { + captureOnlyPrimaryScreen = enabled + UserDefaults.standard.set(enabled, forKey: "captureOnlyPrimaryScreen") + } + + /// JPEG compression quality for screenshots sent to Claude. + /// Lower values reduce payload size (faster uploads) but not token count. + /// Range: 0.0 (most compression) to 1.0 (least compression). Default: 0.8. + @Published var screenshotJPEGQuality: Double = UserDefaults.standard.object(forKey: "screenshotJPEGQuality") == nil + ? 0.8 + : UserDefaults.standard.double(forKey: "screenshotJPEGQuality") + + func setScreenshotJPEGQuality(_ quality: Double) { + screenshotJPEGQuality = quality + UserDefaults.standard.set(quality, forKey: "screenshotJPEGQuality") + } + + /// When enabled, captures only the frontmost application window instead + /// of the entire screen. Useful for reducing noise in screenshots. + @Published var captureActiveWindowOnly: Bool = UserDefaults.standard.bool(forKey: "captureActiveWindowOnly") + + func setCaptureActiveWindowOnly(_ enabled: Bool) { + captureActiveWindowOnly = enabled + UserDefaults.standard.set(enabled, forKey: "captureActiveWindowOnly") + } + /// Whether the user has completed onboarding at least once. Persisted /// to UserDefaults so the Start button only appears on first launch. var hasCompletedOnboarding: Bool { @@ -592,8 +625,12 @@ final class CompanionManager: ObservableObject { voiceState = .processing do { - // Capture all connected screens so the AI has full context - let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG() + // Capture screens using the user's screenshot settings + let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG( + captureOnlyPrimaryScreen: captureOnlyPrimaryScreen, + captureActiveWindowOnly: captureActiveWindowOnly, + jpegCompressionQuality: CGFloat(screenshotJPEGQuality) + ) guard !Task.isCancelled else { return } @@ -970,7 +1007,11 @@ final class CompanionManager: ObservableObject { Task { do { - let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG() + let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG( + captureOnlyPrimaryScreen: true, + captureActiveWindowOnly: captureActiveWindowOnly, + jpegCompressionQuality: CGFloat(screenshotJPEGQuality) + ) // Only send the cursor screen so Claude can't pick something // on a different monitor that we can't point at. diff --git a/leanring-buddy/CompanionPanelView.swift b/leanring-buddy/CompanionPanelView.swift index 76789b4c..cb95f321 100644 --- a/leanring-buddy/CompanionPanelView.swift +++ b/leanring-buddy/CompanionPanelView.swift @@ -58,6 +58,14 @@ struct CompanionPanelView: View { // .padding(.horizontal, 16) // } + if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { + Spacer() + .frame(height: 12) + + screenshotSettingsSection + .padding(.horizontal, 16) + } + if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted { Spacer() .frame(height: 16) @@ -641,6 +649,112 @@ struct CompanionPanelView: View { .pointerCursor() } + // MARK: - Screenshot Settings + + private var screenshotSettingsSection: some View { + VStack(spacing: 2) { + Text("SCREENSHOT") + .font(.system(size: 10, weight: .semibold, design: .rounded)) + .foregroundColor(DS.Colors.textTertiary) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.bottom, 6) + + primaryScreenOnlyToggleRow + + activeWindowOnlyToggleRow + + jpegQualitySliderRow + } + } + + private var primaryScreenOnlyToggleRow: some View { + HStack { + HStack(spacing: 8) { + Image(systemName: "display") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + .frame(width: 16) + + Text("Primary screen only") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + + Spacer() + + Toggle("", isOn: Binding( + get: { companionManager.captureOnlyPrimaryScreen }, + set: { companionManager.setCaptureOnlyPrimaryScreen($0) } + )) + .toggleStyle(.switch) + .labelsHidden() + .tint(DS.Colors.accent) + .scaleEffect(0.8) + } + .padding(.vertical, 4) + } + + private var activeWindowOnlyToggleRow: some View { + HStack { + HStack(spacing: 8) { + Image(systemName: "macwindow") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + .frame(width: 16) + + Text("Active window only") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + + Spacer() + + Toggle("", isOn: Binding( + get: { companionManager.captureActiveWindowOnly }, + set: { companionManager.setCaptureActiveWindowOnly($0) } + )) + .toggleStyle(.switch) + .labelsHidden() + .tint(DS.Colors.accent) + .scaleEffect(0.8) + } + .padding(.vertical, 4) + } + + private var jpegQualitySliderRow: some View { + VStack(spacing: 6) { + HStack { + HStack(spacing: 8) { + Image(systemName: "photo") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(DS.Colors.textTertiary) + .frame(width: 16) + + Text("Image quality") + .font(.system(size: 13, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + } + + Spacer() + + Text("\(Int(companionManager.screenshotJPEGQuality * 100))%") + .font(.system(size: 11, weight: .medium, design: .monospaced)) + .foregroundColor(DS.Colors.textTertiary) + } + + Slider( + value: Binding( + get: { companionManager.screenshotJPEGQuality }, + set: { companionManager.setScreenshotJPEGQuality($0) } + ), + in: 0.3...1.0, + step: 0.1 + ) + .tint(DS.Colors.accent) + } + .padding(.vertical, 4) + } + // MARK: - DM Farza Button private var dmFarzaButton: some View { diff --git a/leanring-buddy/CompanionScreenCaptureUtility.swift b/leanring-buddy/CompanionScreenCaptureUtility.swift index 79784178..5e5c568c 100644 --- a/leanring-buddy/CompanionScreenCaptureUtility.swift +++ b/leanring-buddy/CompanionScreenCaptureUtility.swift @@ -24,10 +24,18 @@ struct CompanionScreenCapture { @MainActor enum CompanionScreenCaptureUtility { - /// Captures all connected displays as JPEG data, labeling each with - /// whether the user's cursor is on that screen. This gives the AI - /// full context across multiple monitors. - static func captureAllScreensAsJPEG() async throws -> [CompanionScreenCapture] { + /// Captures displays as JPEG data based on the provided settings. + /// + /// - Parameters: + /// - captureOnlyPrimaryScreen: When true, only the screen containing the cursor is captured. + /// - captureActiveWindowOnly: When true, captures only the frontmost application window + /// instead of the full screen. + /// - jpegCompressionQuality: JPEG compression factor from 0.0 (max compression) to 1.0 (min compression). + static func captureAllScreensAsJPEG( + captureOnlyPrimaryScreen: Bool = false, + captureActiveWindowOnly: Bool = false, + jpegCompressionQuality: CGFloat = 0.8 + ) async throws -> [CompanionScreenCapture] { let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) guard !content.displays.isEmpty else { @@ -67,6 +75,16 @@ enum CompanionScreenCaptureUtility { return false } + // If capturing only the active window, find the frontmost non-own window + let activeWindow: SCWindow? = captureActiveWindowOnly + ? content.windows.first(where: { window in + window.owningApplication?.bundleIdentifier != ownBundleIdentifier + && window.isOnScreen + && window.frame.width > 100 + && window.frame.height > 100 + }) + : nil + var capturedScreens: [CompanionScreenCapture] = [] for (displayIndex, display) in sortedDisplays.enumerated() { @@ -78,17 +96,43 @@ enum CompanionScreenCaptureUtility { width: CGFloat(display.width), height: CGFloat(display.height)) let isCursorScreen = displayFrame.contains(mouseLocation) - let filter = SCContentFilter(display: display, excludingWindows: ownAppWindows) + // Skip non-cursor screens when primary-only mode is enabled + if captureOnlyPrimaryScreen && !isCursorScreen { + continue + } + + let filter: SCContentFilter + if let activeWindow { + // Capture just the active window — no desktop or other windows + filter = SCContentFilter(desktopIndependentWindow: activeWindow) + } else { + filter = SCContentFilter(display: display, excludingWindows: ownAppWindows) + } let configuration = SCStreamConfiguration() let maxDimension = 1280 - let aspectRatio = CGFloat(display.width) / CGFloat(display.height) - if display.width >= display.height { - configuration.width = maxDimension - configuration.height = Int(CGFloat(maxDimension) / aspectRatio) + + if let activeWindow { + // Size the capture to the window's actual dimensions, capped at maxDimension + let windowWidth = Int(activeWindow.frame.width) + let windowHeight = Int(activeWindow.frame.height) + let windowAspectRatio = CGFloat(windowWidth) / CGFloat(windowHeight) + if windowWidth >= windowHeight { + configuration.width = min(windowWidth, maxDimension) + configuration.height = Int(CGFloat(configuration.width) / windowAspectRatio) + } else { + configuration.height = min(windowHeight, maxDimension) + configuration.width = Int(CGFloat(configuration.height) * windowAspectRatio) + } } else { - configuration.height = maxDimension - configuration.width = Int(CGFloat(maxDimension) * aspectRatio) + let aspectRatio = CGFloat(display.width) / CGFloat(display.height) + if display.width >= display.height { + configuration.width = maxDimension + configuration.height = Int(CGFloat(maxDimension) / aspectRatio) + } else { + configuration.height = maxDimension + configuration.width = Int(CGFloat(maxDimension) * aspectRatio) + } } let cgImage = try await SCScreenshotManager.captureImage( @@ -97,12 +141,15 @@ enum CompanionScreenCaptureUtility { ) guard let jpegData = NSBitmapImageRep(cgImage: cgImage) - .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else { + .representation(using: .jpeg, properties: [.compressionFactor: jpegCompressionQuality]) else { continue } let screenLabel: String - if sortedDisplays.count == 1 { + if captureActiveWindowOnly, let activeWindow { + let appName = activeWindow.owningApplication?.applicationName ?? "unknown app" + screenLabel = "active window (\(appName)) — cursor screen" + } else if sortedDisplays.count == 1 || captureOnlyPrimaryScreen { screenLabel = "user's screen (cursor is here)" } else if isCursorScreen { screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — cursor is on this screen (primary focus)" @@ -120,6 +167,11 @@ enum CompanionScreenCaptureUtility { screenshotWidthInPixels: configuration.width, screenshotHeightInPixels: configuration.height )) + + // Only need one screen when capturing the active window + if captureActiveWindowOnly { + break + } } guard !capturedScreens.isEmpty else { diff --git a/leanring-buddyTests/ScreenshotSettingsTests.swift b/leanring-buddyTests/ScreenshotSettingsTests.swift new file mode 100644 index 00000000..9f33ddec --- /dev/null +++ b/leanring-buddyTests/ScreenshotSettingsTests.swift @@ -0,0 +1,255 @@ +// +// ScreenshotSettingsTests.swift +// leanring-buddyTests +// +// Tests for the screenshot capture settings: primary screen only, +// active window only, and JPEG compression quality. +// + +import Testing +import Foundation +@testable import leanring_buddy + +// MARK: - Screenshot Settings Defaults + +struct ScreenshotSettingsDefaultsTests { + + /// All three screenshot settings should default to their documented values + /// when no UserDefaults entry exists. + + @Test func captureOnlyPrimaryScreenDefaultsToFalse() { + let defaultsKey = "captureOnlyPrimaryScreen" + let savedValue = UserDefaults.standard.object(forKey: defaultsKey) + // Clean slate — remove any leftover test value, check default, then restore + UserDefaults.standard.removeObject(forKey: defaultsKey) + defer { + if let savedValue { + UserDefaults.standard.set(savedValue, forKey: defaultsKey) + } else { + UserDefaults.standard.removeObject(forKey: defaultsKey) + } + } + + let defaultValue = UserDefaults.standard.bool(forKey: defaultsKey) + #expect(defaultValue == false, "captureOnlyPrimaryScreen should default to false (capture all screens)") + } + + @Test func captureActiveWindowOnlyDefaultsToFalse() { + let defaultsKey = "captureActiveWindowOnly" + let savedValue = UserDefaults.standard.object(forKey: defaultsKey) + UserDefaults.standard.removeObject(forKey: defaultsKey) + defer { + if let savedValue { + UserDefaults.standard.set(savedValue, forKey: defaultsKey) + } else { + UserDefaults.standard.removeObject(forKey: defaultsKey) + } + } + + let defaultValue = UserDefaults.standard.bool(forKey: defaultsKey) + #expect(defaultValue == false, "captureActiveWindowOnly should default to false (capture full screen)") + } + + @Test func screenshotJPEGQualityDefaultsTo0Point8() { + let defaultsKey = "screenshotJPEGQuality" + let savedValue = UserDefaults.standard.object(forKey: defaultsKey) + UserDefaults.standard.removeObject(forKey: defaultsKey) + defer { + if let savedValue { + UserDefaults.standard.set(savedValue, forKey: defaultsKey) + } else { + UserDefaults.standard.removeObject(forKey: defaultsKey) + } + } + + // When the key doesn't exist, the code checks for nil and falls back to 0.8 + let keyExists = UserDefaults.standard.object(forKey: defaultsKey) != nil + #expect(keyExists == false, "Key should not exist after removal") + } +} + +// MARK: - Screenshot Settings Persistence + +struct ScreenshotSettingsPersistenceTests { + + @Test func captureOnlyPrimaryScreenPersistsToUserDefaults() { + let defaultsKey = "captureOnlyPrimaryScreen" + let originalValue = UserDefaults.standard.object(forKey: defaultsKey) + defer { + if let originalValue { + UserDefaults.standard.set(originalValue, forKey: defaultsKey) + } else { + UserDefaults.standard.removeObject(forKey: defaultsKey) + } + } + + UserDefaults.standard.set(true, forKey: defaultsKey) + #expect(UserDefaults.standard.bool(forKey: defaultsKey) == true) + + UserDefaults.standard.set(false, forKey: defaultsKey) + #expect(UserDefaults.standard.bool(forKey: defaultsKey) == false) + } + + @Test func captureActiveWindowOnlyPersistsToUserDefaults() { + let defaultsKey = "captureActiveWindowOnly" + let originalValue = UserDefaults.standard.object(forKey: defaultsKey) + defer { + if let originalValue { + UserDefaults.standard.set(originalValue, forKey: defaultsKey) + } else { + UserDefaults.standard.removeObject(forKey: defaultsKey) + } + } + + UserDefaults.standard.set(true, forKey: defaultsKey) + #expect(UserDefaults.standard.bool(forKey: defaultsKey) == true) + + UserDefaults.standard.set(false, forKey: defaultsKey) + #expect(UserDefaults.standard.bool(forKey: defaultsKey) == false) + } + + @Test func screenshotJPEGQualityPersistsToUserDefaults() { + let defaultsKey = "screenshotJPEGQuality" + let originalValue = UserDefaults.standard.object(forKey: defaultsKey) + defer { + if let originalValue { + UserDefaults.standard.set(originalValue, forKey: defaultsKey) + } else { + UserDefaults.standard.removeObject(forKey: defaultsKey) + } + } + + UserDefaults.standard.set(0.5, forKey: defaultsKey) + #expect(UserDefaults.standard.double(forKey: defaultsKey) == 0.5) + + UserDefaults.standard.set(1.0, forKey: defaultsKey) + #expect(UserDefaults.standard.double(forKey: defaultsKey) == 1.0) + + UserDefaults.standard.set(0.3, forKey: defaultsKey) + #expect(UserDefaults.standard.double(forKey: defaultsKey) == 0.3) + } +} + +// MARK: - CompanionScreenCapture Label Tests + +struct ScreenCaptureLabelTests { + + /// Verify that CompanionScreenCapture stores all fields correctly. + @Test func companionScreenCaptureStoresAllFields() { + let testData = Data([0xFF, 0xD8, 0xFF, 0xE0]) // JPEG magic bytes + let testFrame = CGRect(x: 0, y: 0, width: 1512, height: 982) + + let capture = CompanionScreenCapture( + imageData: testData, + label: "user's screen (cursor is here)", + isCursorScreen: true, + displayWidthInPoints: 1512, + displayHeightInPoints: 982, + displayFrame: testFrame, + screenshotWidthInPixels: 1280, + screenshotHeightInPixels: 831 + ) + + #expect(capture.imageData == testData) + #expect(capture.label == "user's screen (cursor is here)") + #expect(capture.isCursorScreen == true) + #expect(capture.displayWidthInPoints == 1512) + #expect(capture.displayHeightInPoints == 982) + #expect(capture.displayFrame == testFrame) + #expect(capture.screenshotWidthInPixels == 1280) + #expect(capture.screenshotHeightInPixels == 831) + } + + @Test func companionScreenCaptureStoresSecondaryScreenLabel() { + let testData = Data([0xFF, 0xD8]) + let testFrame = CGRect(x: 1512, y: 0, width: 2560, height: 1440) + + let capture = CompanionScreenCapture( + imageData: testData, + label: "screen 2 of 2 — secondary screen", + isCursorScreen: false, + displayWidthInPoints: 2560, + displayHeightInPoints: 1440, + displayFrame: testFrame, + screenshotWidthInPixels: 1280, + screenshotHeightInPixels: 720 + ) + + #expect(capture.isCursorScreen == false) + #expect(capture.label.contains("secondary")) + } + + @Test func companionScreenCaptureStoresActiveWindowLabel() { + let testData = Data([0xFF, 0xD8]) + let testFrame = CGRect(x: 0, y: 0, width: 1512, height: 982) + + let capture = CompanionScreenCapture( + imageData: testData, + label: "active window (Safari) — cursor screen", + isCursorScreen: true, + displayWidthInPoints: 1512, + displayHeightInPoints: 982, + displayFrame: testFrame, + screenshotWidthInPixels: 1024, + screenshotHeightInPixels: 768 + ) + + #expect(capture.label.contains("active window")) + #expect(capture.label.contains("Safari")) + } +} + +// MARK: - JPEG Quality Boundary Tests + +struct JPEGQualityBoundaryTests { + + /// The slider range is 0.3–1.0. Verify that values within this range + /// round-trip through UserDefaults correctly. + @Test func qualityAtMinimumSliderBound() { + let defaultsKey = "screenshotJPEGQuality" + let originalValue = UserDefaults.standard.object(forKey: defaultsKey) + defer { + if let originalValue { + UserDefaults.standard.set(originalValue, forKey: defaultsKey) + } else { + UserDefaults.standard.removeObject(forKey: defaultsKey) + } + } + + UserDefaults.standard.set(0.3, forKey: defaultsKey) + let retrieved = UserDefaults.standard.double(forKey: defaultsKey) + #expect(abs(retrieved - 0.3) < 0.001, "Minimum slider value should persist accurately") + } + + @Test func qualityAtMaximumSliderBound() { + let defaultsKey = "screenshotJPEGQuality" + let originalValue = UserDefaults.standard.object(forKey: defaultsKey) + defer { + if let originalValue { + UserDefaults.standard.set(originalValue, forKey: defaultsKey) + } else { + UserDefaults.standard.removeObject(forKey: defaultsKey) + } + } + + UserDefaults.standard.set(1.0, forKey: defaultsKey) + let retrieved = UserDefaults.standard.double(forKey: defaultsKey) + #expect(retrieved == 1.0, "Maximum slider value should persist accurately") + } + + @Test func qualityAtMidpoint() { + let defaultsKey = "screenshotJPEGQuality" + let originalValue = UserDefaults.standard.object(forKey: defaultsKey) + defer { + if let originalValue { + UserDefaults.standard.set(originalValue, forKey: defaultsKey) + } else { + UserDefaults.standard.removeObject(forKey: defaultsKey) + } + } + + UserDefaults.standard.set(0.6, forKey: defaultsKey) + let retrieved = UserDefaults.standard.double(forKey: defaultsKey) + #expect(abs(retrieved - 0.6) < 0.001, "Midpoint slider value should persist accurately") + } +}