diff --git a/src/components/app/app.ts b/src/components/app/app.ts index fade006..58fcc75 100644 --- a/src/components/app/app.ts +++ b/src/components/app/app.ts @@ -32,6 +32,8 @@ import type { import { EuphonySearchWindow } from '../search-window/search-window'; import { NightjarToast } from '../toast/toast'; import { EuphonyTokenWindow } from '../token-window/token-window'; +import type { LocalDataWorkerMessage } from './local-data-worker'; +import LocalDataWorkerInline from './local-data-worker?worker'; import { RequestWorker } from './request-worker'; import { URLManager } from './url-manager'; @@ -276,6 +278,19 @@ export class EuphonyApp extends LitElement { // URL manager urlManager: URLManager; + localDataWorker: Worker; + localDataWorkerRequestCount = 0; + get localDataWorkerRequestID() { + return this.localDataWorkerRequestCount++; + } + activeLocalDataWorkerRequestID: number | null = null; + localDataWorkerPendingRequests = new Map< + number, + { + resolve: () => void; + reject: (reason?: unknown) => void; + } + >(); // Debouncers cacheInfoTooltipDebouncer: number | null = null; @@ -287,6 +302,13 @@ export class EuphonyApp extends LitElement { super(); this.urlManager = new URLManager(this); + this.localDataWorker = new LocalDataWorkerInline(); + this.localDataWorker.addEventListener( + 'message', + (e: MessageEvent) => { + this.localDataWorkerMessageHandler(e); + } + ); // Update the configs based on the current URL this.urlManager.updateConfigsFromURL(); @@ -380,6 +402,11 @@ export class EuphonyApp extends LitElement { }); } + disconnectedCallback(): void { + this.localDataWorker.terminate(); + super.disconnectedCallback(); + } + /** * This method is called when the DOM is added for the first time */ @@ -786,12 +813,14 @@ export class EuphonyApp extends LitElement { break; } case 'Load from clipboard': { + this.isLoadingData = true; navigator.clipboard.readText().then( - clipText => { - this.loadDataFromText(clipText, 'clipboard'); + async clipText => { + await this.loadDataFromText(clipText, 'clipboard'); }, (err: unknown) => { console.error('Failed to read clipboard contents: ', err); + this.isLoadingData = false; } ); break; @@ -1300,125 +1329,147 @@ export class EuphonyApp extends LitElement { } }; - loadDataFromText = (sourceText: string, sourceName: 'clipboard' | 'file') => { - let allData: (Record | string | Conversation)[] = []; - // Try to convert the source text to a JSON or JSONL file. - try { - const jsonData = JSON.parse(sourceText) as Record; - allData = [jsonData]; - } catch (_error) { - // Try to read each line as a JSON object - for (const line of sourceText.split('\n')) { - try { - allData.push(JSON.parse(line) as Record | string); - } catch (_error) { - // pass + loadDataFromText = ( + sourceText: string, + sourceName: 'clipboard' | 'file' + ) => { + this.curPage = 1; + this.resetHash(); + const requestID = this.localDataWorkerRequestID; + this.activeLocalDataWorkerRequestID = requestID; + + return new Promise((resolve, reject) => { + this.localDataWorkerPendingRequests.set(requestID, { resolve, reject }); + const message: LocalDataWorkerMessage = { + command: 'startParseData', + payload: { + requestID, + sourceName, + sourceText } - } - } + }; + this.localDataWorker.postMessage(message); + }); + }; - // Return if there is no data read - if (allData.length === 0) { - this.toastMessage = `Failed to read any JSON or JSONL data from your ${sourceName}. Please double check and try again.`; - this.toastType = 'error'; - if (this.toastComponent) { - this.toastComponent.show(); - } - return; - } + loadDataFromFile = (sourceFile: File) => { + this.curPage = 1; + this.resetHash(); + const requestID = this.localDataWorkerRequestID; + this.activeLocalDataWorkerRequestID = requestID; + + return new Promise((resolve, reject) => { + this.localDataWorkerPendingRequests.set(requestID, { resolve, reject }); + const message: LocalDataWorkerMessage = { + command: 'startParseData', + payload: { + requestID, + sourceName: 'file', + sourceFile + } + }; + this.localDataWorker.postMessage(message); + }); + }; - this.codexSessionData = []; + localDataWorkerMessageHandler(e: MessageEvent) { + switch (e.data.command) { + case 'finishParseData': { + const { requestID, sourceName, dataType } = e.data.payload; + const pendingRequest = + this.localDataWorkerPendingRequests.get(requestID); + this.localDataWorkerPendingRequests.delete(requestID); + if (requestID !== this.activeLocalDataWorkerRequestID) { + pendingRequest?.resolve(); + break; + } + blobPath = null; + this.isLoadingData = false; - // Codex session JSONL is a stream of event objects, not Harmony - // conversations. Detect it early and render with the Codex component. - if (isCodexSessionJSONL(allData as unknown[])) { - this.codexSessionData = [allData as unknown[]]; - this.allConversationData = []; - this.conversationData = []; - this.JSONData = []; - this.selectedConversationIDs = new Set(); - this.dataType = DataType.CODEX; - this._totalConversationSize = 1; - this._totalConversationSizeIncludingUnfiltered = 1; - this.isLoadingFromCache = false; - this.isLoadingFromClipboard = true; - - this.toastMessage = `Codex session loaded successfully from ${sourceName}`; - this.toastType = 'success'; - if (this.toastComponent) { - this.toastComponent.show(); - } - return; - } + this.codexSessionData = []; + this.allConversationData = []; + this.conversationData = []; + this.JSONData = []; - // Validate the data - // If the data is not a conversation, we render it as JSON - if (!this.validateAndTransformConversations(allData)) { - this.toastMessage = - 'Failed to find harmony-formatted data. Render JSON instead.'; - this.toastType = 'warning'; - if (this.toastComponent) { - this.toastComponent.show(); - } + if (dataType === 'codex') { + this.codexSessionData = [e.data.payload.codexSessionData]; + this.selectedConversationIDs = new Set(); + this.dataType = DataType.CODEX; + this._totalConversationSize = 1; + this._totalConversationSizeIncludingUnfiltered = 1; + this.isLoadingFromCache = false; + this.isLoadingFromClipboard = true; - this.JSONData = allData as Record[]; - this.dataType = DataType.JSON; - this._totalConversationSize = allData.length; - this._totalConversationSizeIncludingUnfiltered = allData.length; - return; - } + this.toastMessage = `Codex session loaded successfully from ${sourceName}`; + this.toastType = 'success'; + } else if (dataType === 'json') { + this.JSONData = e.data.payload.jsonData; + this.dataType = DataType.JSON; + this._totalConversationSize = this.JSONData.length; + this._totalConversationSizeIncludingUnfiltered = this.JSONData.length; + this.isLoadingFromCache = false; + this.isLoadingFromClipboard = true; + + this.toastMessage = + 'Failed to find harmony-formatted data. Render JSON instead.'; + this.toastType = 'warning'; + } else { + const conversationData = e.data.payload.conversationData; + this._totalConversationSize = conversationData.length; + this._totalConversationSizeIncludingUnfiltered = + conversationData.length; + + if (this.isEditorMode) { + this.selectedConversationIDs = new Set(); + for (let i = 0; i < conversationData.length; i++) { + this.selectedConversationIDs.add(i); + } + } + + this.allConversationData = conversationData; + this.conversationData = this.isEditorMode + ? conversationData + : conversationData.slice( + (this.curPage - 1) * this.itemsPerPage, + this.curPage * this.itemsPerPage + ); + this.dataType = DataType.CONVERSATION; + this.isLoadingFromCache = false; + this.isLoadingFromClipboard = true; - // The data is valid conversation, so we render it as conversations - this._totalConversationSize = allData.length; - this._totalConversationSizeIncludingUnfiltered = allData.length; + this.toastMessage = `Data loaded successfully from ${sourceName}`; + this.toastType = 'success'; + } - // Set all the conversations as selected in editor mode - if (this.isEditorMode) { - this.selectedConversationIDs = new Set(); - for (let i = 0; i < allData.length; i++) { - this.selectedConversationIDs.add(i); + this.toastComponent?.show(); + pendingRequest?.resolve(); + break; } - } - // People might encode the JSON differently, so we need to load them based - // on the type - if (typeof allData[0] === 'string') { - const newData: Conversation[] = allData.map(item => { - if (typeof item === 'string') { - const parsed = parseConversationJSONString(item); - if (parsed === null) { - this.toastMessage = `Failed to format JSONL data from your ${sourceName}. Please double check and try again.`; - this.toastType = 'error'; - if (this.toastComponent) { - this.toastComponent.show(); - } - throw new Error('Failed to parse conversation JSON string'); - } - return parsed; + case 'error': { + const { requestID, sourceName, message } = e.data.payload; + const pendingRequest = + this.localDataWorkerPendingRequests.get(requestID); + this.localDataWorkerPendingRequests.delete(requestID); + if (requestID !== this.activeLocalDataWorkerRequestID) { + pendingRequest?.reject(new Error(message)); + break; } - return item as Conversation; - }); - this.allConversationData = newData; - this.conversationData = newData; - this.dataType = DataType.CONVERSATION; - } else { - const typedData = allData as Conversation[]; - this.allConversationData = typedData; - this.conversationData = typedData; - this.dataType = DataType.CONVERSATION; - } + this.isLoadingData = false; - // Update the cache info - this.isLoadingFromCache = false; - this.isLoadingFromClipboard = true; + this.toastMessage = `Failed to read any JSON or JSONL data from your ${sourceName}. Please double check and try again.\n\n${message}`; + this.toastType = 'error'; + this.toastComponent?.show(); + pendingRequest?.reject(new Error(message)); + break; + } - // Show a successful toast - this.toastMessage = `Data loaded successfully from ${sourceName}`; - this.toastType = 'success'; - if (this.toastComponent) { - this.toastComponent.show(); + default: { + console.error('Unknown local data worker message', e.data.command); + break; + } } - }; + } localFileInputChanged(e: Event) { const inputElement = e.target as HTMLInputElement; @@ -1427,15 +1478,13 @@ export class EuphonyApp extends LitElement { return; } - file - .text() - .then(text => { - this.loadDataFromText(text, 'file'); - }) + this.isLoadingData = true; + this.loadDataFromFile(file) .catch((error: unknown) => { this.toastMessage = `Failed to read local file.\n\n${error}`; this.toastType = 'error'; this.toastComponent?.show(); + this.isLoadingData = false; }) .finally(() => { inputElement.value = ''; diff --git a/src/components/app/local-data-worker.ts b/src/components/app/local-data-worker.ts new file mode 100644 index 0000000..ca443bf --- /dev/null +++ b/src/components/app/local-data-worker.ts @@ -0,0 +1,279 @@ +import type { Conversation } from '../../types/harmony-types'; +import { isCodexSessionJSONL } from '../../utils/codex-session'; + +type ParsedItem = Record | string | Conversation; + +export type LocalDataWorkerMessage = + | { + command: 'startParseData'; + payload: { + requestID: number; + sourceName: 'clipboard' | 'file'; + sourceText?: string; + sourceFile?: File; + }; + } + | { + command: 'finishParseData'; + payload: + | { + requestID: number; + sourceName: 'clipboard' | 'file'; + dataType: 'codex'; + codexSessionData: unknown[]; + } + | { + requestID: number; + sourceName: 'clipboard' | 'file'; + dataType: 'conversation'; + conversationData: Conversation[]; + } + | { + requestID: number; + sourceName: 'clipboard' | 'file'; + dataType: 'json'; + jsonData: Record[]; + }; + } + | { + command: 'error'; + payload: { + requestID: number; + sourceName: 'clipboard' | 'file'; + message: string; + }; + }; + +const isConversation = (data: unknown) => { + if (typeof data !== 'object' || data === null) { + return false; + } + return 'messages' in data && Array.isArray(data.messages); +}; + +const extractConversationFromJSONL = ( + data: unknown[] +): Conversation[] | null => { + let curData: Record[] | null = null; + + if ( + data.length > 0 && + typeof data[0] === 'object' && + !isConversation(data[0]) + ) { + curData = data as Record[]; + } + + if (data.length > 0 && typeof data[0] === 'string') { + let shouldSkipTransformation = false; + try { + const conversation = JSON.parse(data[0]) as Conversation; + if (isConversation(conversation)) { + shouldSkipTransformation = true; + } + } catch (_error) { + shouldSkipTransformation = true; + } + + if (!shouldSkipTransformation) { + curData = []; + for (const d of data) { + const record = JSON.parse(d as string) as Record< + string, + Conversation | string + >; + curData.push(record); + } + } + } + + if (curData !== null) { + let conversationKey: string | null = null; + let conversationFieldIsString = false; + + for (const key in curData[0]) { + if (typeof curData[0][key] === 'string') { + try { + const conversation = JSON.parse(curData[0][key]) as Conversation; + if (isConversation(conversation)) { + conversationKey = key; + conversationFieldIsString = true; + break; + } + } catch (_error) { + continue; + } + } else if (isConversation(curData[0][key])) { + conversationKey = key; + break; + } + } + + if (conversationKey !== null) { + const conversationData: Conversation[] = []; + + for (const d of curData) { + const conversation = conversationFieldIsString + ? (JSON.parse(d[conversationKey] as string) as Conversation) + : (d[conversationKey] as Conversation); + conversation.metadata ??= {}; + + for (const k in d) { + if (k !== conversationKey) { + conversation.metadata[`euphonyTransformed-${k}`] = d[k]; + } + } + + conversationData.push(conversation); + } + return conversationData; + } + } + + return null; +}; + +const validateAndTransformConversations = ( + conversations: ParsedItem[] +): conversations is Conversation[] => { + const allValid: boolean[] = []; + + for (const [i, conversation] of conversations.entries()) { + if (typeof conversation === 'string') { + const conversationData = JSON.parse(conversation) as Record< + string, + unknown + >; + let newItem = conversation; + + if ( + conversationData.conversation_id !== undefined && + conversationData.id === undefined + ) { + conversationData.id = conversationData.conversation_id; + newItem = JSON.stringify(conversationData); + } + + conversations[i] = newItem; + allValid.push(Array.isArray(conversationData.messages)); + } else { + const conversationData = conversation as Record; + + if ( + conversationData.conversation_id !== undefined && + conversationData.id === undefined + ) { + conversationData.id = conversationData.conversation_id; + } + + conversations[i] = conversationData; + allValid.push(Array.isArray(conversationData.messages)); + } + } + + return allValid.every(Boolean); +}; + +const parseSourceText = (sourceText: string): ParsedItem[] => { + const allData: ParsedItem[] = []; + + try { + const jsonData = JSON.parse(sourceText) as Record; + allData.push(jsonData); + return allData; + } catch (_error) { + for (const line of sourceText.split('\n')) { + try { + allData.push(JSON.parse(line) as Record | string); + } catch (_innerError) { + // Skip invalid JSONL lines. + } + } + } + + return allData; +}; + +const parseLocalData = ( + sourceText: string +): + | { dataType: 'codex'; codexSessionData: unknown[] } + | { dataType: 'conversation'; conversationData: Conversation[] } + | { dataType: 'json'; jsonData: Record[] } => { + let allData = parseSourceText(sourceText); + + if (allData.length === 0) { + throw new Error('Failed to read any JSON or JSONL data.'); + } + + if (isCodexSessionJSONL(allData as unknown[])) { + return { + dataType: 'codex', + codexSessionData: allData as unknown[] + }; + } + + const transformedConversationData = extractConversationFromJSONL( + allData as unknown[] + ); + if (transformedConversationData) { + allData = transformedConversationData; + } + + if (!validateAndTransformConversations(allData)) { + return { + dataType: 'json', + jsonData: allData as Record[] + }; + } + + const conversationData: Conversation[] = []; + for (const item of allData) { + if (typeof item === 'string') { + conversationData.push(JSON.parse(item) as Conversation); + } else { + conversationData.push(item); + } + } + + return { + dataType: 'conversation', + conversationData + }; +}; + +self.onmessage = async (e: MessageEvent) => { + if (e.data.command !== 'startParseData') { + console.error('Worker: unknown message', e.data.command); + return; + } + + const { requestID, sourceName, sourceText, sourceFile } = e.data.payload; + + try { + const text = sourceText ?? (await sourceFile?.text()); + if (text === undefined) { + throw new Error('No source text or file was provided.'); + } + const result = parseLocalData(text); + const message: LocalDataWorkerMessage = { + command: 'finishParseData', + payload: { + requestID, + sourceName, + ...result + } + }; + postMessage(message); + } catch (error) { + const message: LocalDataWorkerMessage = { + command: 'error', + payload: { + requestID, + sourceName, + message: error instanceof Error ? error.message : String(error) + } + }; + postMessage(message); + } +}; diff --git a/src/utils/api-manager.ts b/src/utils/api-manager.ts index c2f256d..97884bd 100644 --- a/src/utils/api-manager.ts +++ b/src/utils/api-manager.ts @@ -46,7 +46,7 @@ const isComparison = (data: unknown) => { return 'conversation' in data && 'completions' in data; }; -const extractConversationFromJSONL = ( +export const extractConversationFromJSONL = ( data: unknown[] ): Conversation[] | null => { // Simple transformation to handle the case where the Conversation is stored