Skip to content

Add KMP-backed local transcription runtime #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
watzon merged 5 commits into from
Mar 29, 2026
Merged

Add KMP-backed local transcription runtime #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e4b314c
Add KMP-backed local transcription runtime
watzon Mar 29, 2026
7504129
Add local transcription session coordination
watzon Mar 29, 2026
55f021a
Add shared whisper.cpp transcription runtime
watzon Mar 29, 2026
55656ad
Implement shared whisper.cpp runtime plumbing
watzon Mar 29, 2026
d06f290
Address PR review feedback
watzon Mar 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ Carthage/Build/
# Temporary scripts and build artifacts
*.py
package-lock.json
*.hprof

DerivedDataTests/

Expand Down
55 changes: 51 additions & 4 deletions Pindrop/Services/ModelManager.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,13 @@ class ModelManager {
private(set) var downloadedFeatureModels: Set<FeatureModelType> = []

private let fileManager = FileManager.default
#if canImport(PindropSharedTranscription)
@ObservationIgnored
private lazy var localRuntimeBridge = KMPTranscriptionRuntimeBridge(
modelManager: self,
engineFactory: TranscriptionService.defaultEngineFactory(provider:)
)
#endif

/// Last decile (0...10) logged for WhisperKit file download progress to avoid log spam.
private var whisperKitDownloadLastLoggedDecile: Int = -1
Expand Down Expand Up @@ -261,6 +268,18 @@ class ModelManager {
}

func refreshDownloadedModels() async {
#if canImport(PindropSharedTranscription)
let downloaded = await localRuntimeBridge.refreshInstalledModelNames()
if downloaded != downloadedModelNames {
Log.model.debug("Found \(downloaded.count) downloaded models via KMP runtime: \(downloaded)")
}
downloadedModelNames = downloaded
#else
await refreshDownloadedModelsFromDisk()
#endif
}

private func refreshDownloadedModelsFromDisk() async {
var downloaded: Set<String> = []

let whisperKitPath = whisperKitModelsURL
Expand Down Expand Up @@ -339,13 +358,31 @@ class ModelManager {
isDownloading = false
currentDownloadModel = nil
}


#if canImport(PindropSharedTranscription)
if model.provider.isLocal {
try await localRuntimeBridge.installModel(named: modelName, onProgress: onProgress)
} else {
throw ModelError.downloadNotImplemented(model.provider.rawValue)
}
#else
try await installModelArtifacts(named: modelName, onProgress: onProgress)
#endif
Log.boot.info("ModelManager.downloadModel finished OK name=\(modelName) wallClock=\(String(format: "%.2fs", CFAbsoluteTimeGetCurrent() - downloadWallClock))")
}

func installModelArtifacts(named modelName: String, onProgress: ((Double) -> Void)? = nil) async throws {
guard let model = availableModels.first(where: { $0.name == modelName }) else {
throw ModelError.modelNotFound(modelName)
}

if model.provider == .parakeet {
try await downloadParakeetModel(named: modelName, onProgress: onProgress)
} else {
} else if model.provider == .whisperKit {
try await downloadWhisperKitModel(named: modelName, onProgress: onProgress)
} else {
throw ModelError.downloadNotImplemented(model.provider.rawValue)
}
Log.boot.info("ModelManager.downloadModel finished OK name=\(modelName) wallClock=\(String(format: "%.2fs", CFAbsoluteTimeGetCurrent() - downloadWallClock))")
}

private func downloadWhisperKitModel(named modelName: String, onProgress: ((Double) -> Void)? = nil) async throws {
Expand Down Expand Up @@ -474,6 +511,17 @@ class ModelManager {
}

func deleteModel(named modelName: String) async throws {
#if canImport(PindropSharedTranscription)
try await localRuntimeBridge.deleteModel(named: modelName)
await refreshDownloadedModels()
return
#else
try await deleteModelArtifacts(named: modelName)
await refreshDownloadedModels()
#endif
}

func deleteModelArtifacts(named modelName: String) async throws {
guard let model = availableModels.first(where: { $0.name == modelName }) else {
throw ModelError.modelNotFound(modelName)
}
Expand All @@ -488,7 +536,6 @@ class ModelManager {

do {
try fileManager.removeItem(at: modelPath)
await refreshDownloadedModels()
} catch {
throw ModelError.deleteFailed(error.localizedDescription)
}
Expand Down
120 changes: 104 additions & 16 deletions Pindrop/Services/Transcription/KMPTranscriptionBridge.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,25 @@ struct SharedTranscriptionStateTransition: Equatable, Sendable {
}

enum KMPTranscriptionBridge {
static func localAvailableModels() -> [ModelManager.WhisperModel] {
#if canImport(PindropSharedTranscription)
LocalTranscriptionCatalog.shared.models(platform: localPlatform()).map(localModel(from:))
#else
[]
#endif
}

static func recommendedLocalModels(for language: AppLanguage) -> [ModelManager.WhisperModel] {
#if canImport(PindropSharedTranscription)
LocalTranscriptionCatalog.shared.recommendedModels(
platform: localPlatform(),
language: coreLanguage(from: language)
).map(localModel(from:))
#else
[]
#endif
}

static func normalizeTranscriptionText(_ text: String) -> String {
#if canImport(PindropSharedTranscription)
SharedTranscriptionOrchestrator.shared.normalizeTranscriptionText(text: text)
Expand Down Expand Up @@ -187,7 +206,7 @@ enum KMPTranscriptionBridge {
#if canImport(PindropSharedTranscription)
let policy = TranscriptionRuntimePolicy(
selectedProvider: coreProvider(from: selectedProvider),
selectedModelId: CoreTranscriptionModelId(value: selectedModelName),
selectedModelId: TranscriptionModelId(value: selectedModelName),
streamingFeatureEnabled: streamingFeatureEnabled,
diarizationFeatureEnabled: diarizationFeatureEnabled,
outputMode: outputMode.kmpValue,
Expand Down Expand Up @@ -256,7 +275,7 @@ enum KMPTranscriptionBridge {
#if canImport(PindropSharedTranscription)
let plan = SharedTranscriptionOrchestrator.shared.planTranscriptionExecution(
selectedProvider: coreProvider(from: selectedProvider),
selectedModelId: CoreTranscriptionModelId(value: selectedModelName),
selectedModelId: TranscriptionModelId(value: selectedModelName),
diarizationRequested: diarizationRequested,
isStreamingSessionActive: isStreamingSessionActive
)
Expand Down Expand Up @@ -441,7 +460,7 @@ enum KMPTranscriptionBridge {
) -> [ModelManager.WhisperModel] {
#if canImport(PindropSharedTranscription)
let orchestrator = SharedTranscriptionOrchestrator.shared
let curatedIds = recommendedModelNames(for: language).map { CoreTranscriptionModelId(value: $0) }
let curatedIds = recommendedModelNames(for: language).map { TranscriptionModelId(value: $0) }
let descriptors = availableModels.map(coreDescriptor(from:))
let language = coreLanguage(from: language)

Expand Down Expand Up @@ -482,10 +501,10 @@ enum KMPTranscriptionBridge {
let modelsByName = Dictionary(uniqueKeysWithValues: availableModels.map { ($0.name, $0) })

let resolution = orchestrator.resolveStartupModel(
selectedModelId: CoreTranscriptionModelId(value: selectedModelId),
defaultModelId: CoreTranscriptionModelId(value: defaultModelId),
selectedModelId: TranscriptionModelId(value: selectedModelId),
defaultModelId: TranscriptionModelId(value: defaultModelId),
availableModels: descriptors,
downloadedModelIds: downloadedModelIds.map(CoreTranscriptionModelId.init(value:))
downloadedModelIds: downloadedModelIds.map { TranscriptionModelId(value: $0) }
)

let resolvedModel = modelsByName[resolution.resolvedModel.id.value] ?? availableModels.first!
Expand Down Expand Up @@ -610,7 +629,7 @@ enum KMPTranscriptionBridge {

#if canImport(PindropSharedTranscription)
private extension KMPTranscriptionBridge {
static func coreProvider(from provider: ModelManager.ModelProvider) -> CoreTranscriptionProviderId {
static func coreProvider(from provider: ModelManager.ModelProvider) -> TranscriptionProviderId {
switch provider {
case .whisperKit:
.whisperKit
Expand All @@ -625,7 +644,7 @@ private extension KMPTranscriptionBridge {
}
}

static func modelProvider(from provider: CoreTranscriptionProviderId) -> ModelManager.ModelProvider {
static func modelProvider(from provider: TranscriptionProviderId) -> ModelManager.ModelProvider {
switch provider {
case .whisperKit:
.whisperKit
Expand All @@ -642,7 +661,7 @@ private extension KMPTranscriptionBridge {
}
}

static func coreLanguage(from language: AppLanguage) -> CoreTranscriptionLanguage {
static func coreLanguage(from language: AppLanguage) -> TranscriptionLanguage {
switch language {
case .automatic:
.automatic
Expand Down Expand Up @@ -673,7 +692,7 @@ private extension KMPTranscriptionBridge {

static func coreLanguageSupport(
from support: ModelManager.LanguageSupport
) -> CoreModelLanguageSupport {
) -> ModelLanguageSupport {
switch support {
case .englishOnly:
.englishOnly
Expand All @@ -686,7 +705,7 @@ private extension KMPTranscriptionBridge {

static func coreAvailability(
from availability: ModelManager.ModelAvailability
) -> CoreModelAvailability {
) -> ModelAvailability {
switch availability {
case .available:
.available
Expand All @@ -697,9 +716,9 @@ private extension KMPTranscriptionBridge {
}
}

static func coreDescriptor(from model: ModelManager.WhisperModel) -> CoreModelDescriptor {
CoreModelDescriptor(
id: CoreTranscriptionModelId(value: model.name),
static func coreDescriptor(from model: ModelManager.WhisperModel) -> ModelDescriptor {
ModelDescriptor(
id: TranscriptionModelId(value: model.name),
displayName: model.displayName,
provider: coreProvider(from: model.provider),
languageSupport: coreLanguageSupport(from: model.languageSupport),
Expand All @@ -711,7 +730,76 @@ private extension KMPTranscriptionBridge {
)
}

static func coreState(from state: TranscriptionService.State) -> CoreSharedTranscriptionState {
static func localProvider(
from provider: LocalModelProvider
) -> ModelManager.ModelProvider {
switch provider {
case .whisperKit, .wcpp:
.whisperKit
case .parakeetCoreml, .parakeetNative:
.parakeet
default:
.whisperKit
}
}

static func localAvailability(
from availability: ModelAvailability
) -> ModelManager.ModelAvailability {
switch availability {
case .available:
.available
case .comingSoon:
.comingSoon
case .requiresSetup:
.requiresSetup
default:
.available
}
}

static func localLanguageSupport(
from support: ModelLanguageSupport
) -> ModelManager.LanguageSupport {
switch support {
case .englishOnly:
.englishOnly
case .fullMultilingual:
.fullMultilingual
case .parakeetV3European:
.parakeetV3European
default:
.fullMultilingual
}
}

static func localModel(
from descriptor: LocalModelDescriptor
) -> ModelManager.WhisperModel {
ModelManager.WhisperModel(
name: descriptor.id.value,
displayName: descriptor.displayName,
sizeInMB: Int(descriptor.sizeInMb),
description: descriptor.description_,
speedRating: descriptor.speedRating,
accuracyRating: descriptor.accuracyRating,
languageSupport: localLanguageSupport(from: descriptor.languageSupport),
provider: localProvider(from: descriptor.provider),
availability: localAvailability(from: descriptor.availability)
)
}

static func localPlatform() -> LocalPlatformId {
#if os(macOS)
.macos
#elseif os(Windows)
.windows
#else
.linux
#endif
}

static func coreState(from state: TranscriptionService.State) -> SharedTranscriptionState {
switch state {
case .unloaded:
.unloaded
Expand All @@ -726,7 +814,7 @@ private extension KMPTranscriptionBridge {
}
}

static func serviceState(from state: CoreSharedTranscriptionState) -> TranscriptionService.State {
static func serviceState(from state: SharedTranscriptionState) -> TranscriptionService.State {
switch state {
case .unloaded:
.unloaded
Expand Down
Loading
Loading